2024-11-20 12:14:18,567 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 12:14:18,582 main DEBUG Took 0.013019 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 12:14:18,583 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 12:14:18,583 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 12:14:18,584 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 12:14:18,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,595 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 12:14:18,609 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,612 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,613 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,615 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,616 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,618 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,619 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,620 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,622 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,623 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,624 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,625 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,625 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:14:18,626 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,626 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 12:14:18,628 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:14:18,630 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 12:14:18,633 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 12:14:18,634 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 12:14:18,635 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 12:14:18,636 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 12:14:18,646 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 12:14:18,649 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 12:14:18,650 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 12:14:18,651 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 12:14:18,651 main DEBUG createAppenders(={Console}) 2024-11-20 12:14:18,652 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 12:14:18,652 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 12:14:18,653 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 12:14:18,653 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 12:14:18,654 main DEBUG OutputStream closed 2024-11-20 12:14:18,654 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 12:14:18,654 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 12:14:18,655 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 12:14:18,735 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 12:14:18,738 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 12:14:18,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 12:14:18,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 12:14:18,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 12:14:18,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 12:14:18,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 12:14:18,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 12:14:18,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 12:14:18,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 12:14:18,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 12:14:18,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 12:14:18,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 12:14:18,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 12:14:18,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 12:14:18,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 12:14:18,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 12:14:18,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 12:14:18,748 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 12:14:18,748 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 12:14:18,749 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 12:14:18,749 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T12:14:18,998 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75 2024-11-20 12:14:19,001 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 12:14:19,001 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T12:14:19,010 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-20T12:14:19,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T12:14:19,048 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d, deleteOnExit=true 2024-11-20T12:14:19,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T12:14:19,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/test.cache.data in system properties and HBase conf 2024-11-20T12:14:19,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T12:14:19,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.log.dir in system properties and HBase conf 2024-11-20T12:14:19,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T12:14:19,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T12:14:19,052 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T12:14:19,131 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T12:14:19,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T12:14:19,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T12:14:19,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T12:14:19,230 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T12:14:19,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T12:14:19,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T12:14:19,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T12:14:19,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T12:14:19,232 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T12:14:19,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T12:14:19,233 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/nfs.dump.dir in system properties and HBase conf 2024-11-20T12:14:19,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/java.io.tmpdir in system properties and HBase conf 2024-11-20T12:14:19,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T12:14:19,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T12:14:19,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T12:14:20,271 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T12:14:20,348 INFO [Time-limited test {}] log.Log(170): Logging initialized @2518ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T12:14:20,416 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:14:20,473 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:14:20,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:14:20,492 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:14:20,494 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T12:14:20,505 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:14:20,508 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:14:20,509 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:14:20,688 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/java.io.tmpdir/jetty-localhost-33089-hadoop-hdfs-3_4_1-tests_jar-_-any-9688262948455459840/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T12:14:20,694 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:33089} 2024-11-20T12:14:20,695 INFO [Time-limited test {}] server.Server(415): Started @2865ms 2024-11-20T12:14:21,253 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:14:21,260 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:14:21,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:14:21,261 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:14:21,262 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T12:14:21,262 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:14:21,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:14:21,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3dfb1097{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/java.io.tmpdir/jetty-localhost-34059-hadoop-hdfs-3_4_1-tests_jar-_-any-9062763903946318141/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T12:14:21,362 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:34059} 2024-11-20T12:14:21,362 INFO [Time-limited test {}] server.Server(415): Started @3533ms 2024-11-20T12:14:21,411 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T12:14:22,595 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data1/current/BP-788036131-172.17.0.3-1732104859763/current, will proceed with Du for space computation calculation, 2024-11-20T12:14:22,595 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data2/current/BP-788036131-172.17.0.3-1732104859763/current, will proceed with Du for space computation calculation, 2024-11-20T12:14:22,635 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T12:14:22,684 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaf37a55880f7d5a with lease ID 0x66b3aa9feca9523e: Processing first storage report for DS-e866e262-95f8-4b3c-ba6f-4b0dc738e012 from datanode DatanodeRegistration(127.0.0.1:39791, datanodeUuid=b670e9a4-6ad7-405c-9da8-9f4952936e79, infoPort=34329, infoSecurePort=0, ipcPort=41431, storageInfo=lv=-57;cid=testClusterID;nsid=1429073397;c=1732104859763) 2024-11-20T12:14:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaf37a55880f7d5a with lease ID 0x66b3aa9feca9523e: from storage DS-e866e262-95f8-4b3c-ba6f-4b0dc738e012 node DatanodeRegistration(127.0.0.1:39791, datanodeUuid=b670e9a4-6ad7-405c-9da8-9f4952936e79, infoPort=34329, infoSecurePort=0, ipcPort=41431, storageInfo=lv=-57;cid=testClusterID;nsid=1429073397;c=1732104859763), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-20T12:14:22,688 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaf37a55880f7d5a with lease ID 0x66b3aa9feca9523e: Processing first storage report for DS-5b43e8cb-56cf-4759-845d-ac71c1895a51 from datanode DatanodeRegistration(127.0.0.1:39791, datanodeUuid=b670e9a4-6ad7-405c-9da8-9f4952936e79, infoPort=34329, infoSecurePort=0, ipcPort=41431, storageInfo=lv=-57;cid=testClusterID;nsid=1429073397;c=1732104859763) 2024-11-20T12:14:22,688 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaf37a55880f7d5a with lease ID 0x66b3aa9feca9523e: from storage DS-5b43e8cb-56cf-4759-845d-ac71c1895a51 node DatanodeRegistration(127.0.0.1:39791, datanodeUuid=b670e9a4-6ad7-405c-9da8-9f4952936e79, infoPort=34329, infoSecurePort=0, ipcPort=41431, storageInfo=lv=-57;cid=testClusterID;nsid=1429073397;c=1732104859763), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T12:14:22,706 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75 2024-11-20T12:14:22,803 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/zookeeper_0, clientPort=61195, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T12:14:22,819 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61195 2024-11-20T12:14:22,833 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:22,841 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:23,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741825_1001 (size=7) 2024-11-20T12:14:23,547 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 with version=8 2024-11-20T12:14:23,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/hbase-staging 2024-11-20T12:14:23,661 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T12:14:23,884 INFO [Time-limited test {}] client.ConnectionUtils(128): master/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:14:23,894 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:23,895 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:23,900 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:14:23,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:23,900 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:14:24,060 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T12:14:24,115 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T12:14:24,125 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T12:14:24,128 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:14:24,158 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4667 (auto-detected) 2024-11-20T12:14:24,159 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-20T12:14:24,183 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40015 2024-11-20T12:14:24,211 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40015 connecting to ZooKeeper ensemble=127.0.0.1:61195 2024-11-20T12:14:24,305 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400150x0, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:14:24,308 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40015-0x101582ab7dc0000 connected 2024-11-20T12:14:24,587 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:24,589 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:24,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:14:24,612 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91, hbase.cluster.distributed=false 2024-11-20T12:14:24,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:14:24,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40015 2024-11-20T12:14:24,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40015 2024-11-20T12:14:24,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40015 2024-11-20T12:14:24,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40015 2024-11-20T12:14:24,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40015 2024-11-20T12:14:24,793 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:14:24,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:24,795 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:24,795 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:14:24,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:14:24,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:14:24,800 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T12:14:24,804 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:14:24,806 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42825 2024-11-20T12:14:24,808 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42825 connecting to ZooKeeper ensemble=127.0.0.1:61195 2024-11-20T12:14:24,809 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:24,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:24,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428250x0, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:14:24,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:428250x0, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:14:24,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42825-0x101582ab7dc0001 connected 2024-11-20T12:14:24,852 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T12:14:24,859 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T12:14:24,862 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T12:14:24,867 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:14:24,868 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-20T12:14:24,869 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42825 2024-11-20T12:14:24,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42825 2024-11-20T12:14:24,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-20T12:14:24,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-20T12:14:24,890 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;843d83d51ba0:40015 2024-11-20T12:14:24,891 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/843d83d51ba0,40015,1732104863721 2024-11-20T12:14:24,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:14:24,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:14:24,961 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/843d83d51ba0,40015,1732104863721 2024-11-20T12:14:25,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T12:14:25,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:25,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:25,036 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T12:14:25,037 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/843d83d51ba0,40015,1732104863721 from backup master directory 2024-11-20T12:14:25,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/843d83d51ba0,40015,1732104863721 2024-11-20T12:14:25,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:14:25,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:14:25,097 WARN [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:14:25,098 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=843d83d51ba0,40015,1732104863721 2024-11-20T12:14:25,099 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T12:14:25,101 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T12:14:25,148 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/hbase.id] with ID: 8050fc35-0ca3-4aaf-8d5d-0952e660f100 2024-11-20T12:14:25,148 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/.tmp/hbase.id 2024-11-20T12:14:25,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741826_1002 (size=42) 2024-11-20T12:14:25,560 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/.tmp/hbase.id]:[hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/hbase.id] 2024-11-20T12:14:25,609 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:14:25,614 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T12:14:25,635 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-20T12:14:25,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:25,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:25,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741827_1003 (size=196) 2024-11-20T12:14:26,143 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:14:26,145 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T12:14:26,159 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:26,163 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:14:26,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741828_1004 (size=1189) 2024-11-20T12:14:26,612 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store 2024-11-20T12:14:26,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741829_1005 (size=34) 2024-11-20T12:14:27,042 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T12:14:27,046 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:27,047 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T12:14:27,047 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:14:27,047 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:14:27,048 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T12:14:27,049 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:14:27,049 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:14:27,050 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732104867047Disabling compacts and flushes for region at 1732104867047Disabling writes for close at 1732104867048 (+1 ms)Writing region close event to WAL at 1732104867049 (+1 ms)Closed at 1732104867049 2024-11-20T12:14:27,052 WARN [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/.initializing 2024-11-20T12:14:27,053 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/WALs/843d83d51ba0,40015,1732104863721 2024-11-20T12:14:27,060 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:14:27,077 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C40015%2C1732104863721, suffix=, logDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/WALs/843d83d51ba0,40015,1732104863721, archiveDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/oldWALs, maxLogs=10 2024-11-20T12:14:27,104 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/WALs/843d83d51ba0,40015,1732104863721/843d83d51ba0%2C40015%2C1732104863721.1732104867082, exclude list is [], retry=0 2024-11-20T12:14:27,124 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39791,DS-e866e262-95f8-4b3c-ba6f-4b0dc738e012,DISK] 2024-11-20T12:14:27,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T12:14:27,176 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/WALs/843d83d51ba0,40015,1732104863721/843d83d51ba0%2C40015%2C1732104863721.1732104867082 2024-11-20T12:14:27,182 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34329:34329)] 2024-11-20T12:14:27,183 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:14:27,184 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:27,188 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,189 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T12:14:27,282 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:27,285 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:27,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,290 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T12:14:27,290 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:27,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:27,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T12:14:27,297 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:27,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:27,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T12:14:27,301 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:27,302 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:27,303 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,307 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,308 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,315 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,315 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,319 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T12:14:27,323 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:14:27,328 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:14:27,329 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59691863, jitterRate=-0.11052192747592926}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T12:14:27,336 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732104867206Initializing all the Stores at 1732104867208 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104867209 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104867210 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104867210Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104867211 (+1 ms)Cleaning up temporary data from old regions at 1732104867316 (+105 ms)Region opened successfully at 1732104867335 (+19 ms) 2024-11-20T12:14:27,337 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T12:14:27,373 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c71b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:14:27,425 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T12:14:27,443 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T12:14:27,443 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T12:14:27,450 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:14:27,452 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-20T12:14:27,462 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T12:14:27,462 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T12:14:27,494 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T12:14:27,503 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T12:14:27,521 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T12:14:27,525 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T12:14:27,527 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T12:14:27,540 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T12:14:27,543 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T12:14:27,547 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T12:14:27,553 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T12:14:27,555 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T12:14:27,606 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T12:14:27,627 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T12:14:27,637 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T12:14:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:14:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:14:27,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,651 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=843d83d51ba0,40015,1732104863721, sessionid=0x101582ab7dc0000, setting cluster-up flag (Was=false) 2024-11-20T12:14:27,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,711 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T12:14:27,713 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=843d83d51ba0,40015,1732104863721 2024-11-20T12:14:27,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:27,774 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T12:14:27,776 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=843d83d51ba0,40015,1732104863721 2024-11-20T12:14:27,782 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T12:14:27,847 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:14:27,857 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T12:14:27,863 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T12:14:27,868 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 843d83d51ba0,40015,1732104863721 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T12:14:27,875 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:14:27,875 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:14:27,875 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:14:27,875 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:14:27,875 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/843d83d51ba0:0, corePoolSize=10, maxPoolSize=10 2024-11-20T12:14:27,876 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:27,876 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:14:27,876 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:27,877 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104897877 2024-11-20T12:14:27,880 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T12:14:27,881 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T12:14:27,882 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:14:27,882 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T12:14:27,886 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T12:14:27,886 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(746): ClusterId : 8050fc35-0ca3-4aaf-8d5d-0952e660f100 2024-11-20T12:14:27,886 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T12:14:27,887 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T12:14:27,887 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T12:14:27,890 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T12:14:27,890 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:27,891 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T12:14:27,892 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:27,900 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T12:14:27,902 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T12:14:27,902 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T12:14:27,903 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T12:14:27,903 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T12:14:27,904 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T12:14:27,905 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T12:14:27,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741831_1007 (size=1321) 2024-11-20T12:14:27,912 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T12:14:27,913 DEBUG [RS:0;843d83d51ba0:42825 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77398160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:14:27,919 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104867906,5,FailOnTimeoutGroup] 2024-11-20T12:14:27,920 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104867919,5,FailOnTimeoutGroup] 2024-11-20T12:14:27,921 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:27,921 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T12:14:27,922 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:27,923 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:27,938 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;843d83d51ba0:42825 2024-11-20T12:14:27,943 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T12:14:27,943 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T12:14:27,943 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T12:14:27,947 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(2659): reportForDuty to master=843d83d51ba0,40015,1732104863721 with port=42825, startcode=1732104864746 2024-11-20T12:14:27,959 DEBUG [RS:0;843d83d51ba0:42825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T12:14:27,997 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53989, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T12:14:28,002 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40015 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,004 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40015 {}] master.ServerManager(517): Registering regionserver=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,019 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:14:28,019 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36391 2024-11-20T12:14:28,019 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T12:14:28,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:14:28,035 DEBUG [RS:0;843d83d51ba0:42825 {}] zookeeper.ZKUtil(111): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,036 WARN [RS:0;843d83d51ba0:42825 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:14:28,036 INFO [RS:0;843d83d51ba0:42825 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:14:28,036 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,039 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [843d83d51ba0,42825,1732104864746] 2024-11-20T12:14:28,064 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T12:14:28,084 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T12:14:28,090 INFO [RS:0;843d83d51ba0:42825 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T12:14:28,090 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,091 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T12:14:28,097 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T12:14:28,098 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,099 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,100 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:14:28,101 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:14:28,101 DEBUG [RS:0;843d83d51ba0:42825 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:14:28,102 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,102 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,102 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,103 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,103 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,103 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,42825,1732104864746-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:14:28,123 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T12:14:28,124 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,42825,1732104864746-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,125 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,125 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.Replication(171): 843d83d51ba0,42825,1732104864746 started 2024-11-20T12:14:28,144 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:28,145 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1482): Serving as 843d83d51ba0,42825,1732104864746, RpcServer on 843d83d51ba0/172.17.0.3:42825, sessionid=0x101582ab7dc0001 2024-11-20T12:14:28,146 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T12:14:28,146 DEBUG [RS:0;843d83d51ba0:42825 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,146 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,42825,1732104864746' 2024-11-20T12:14:28,146 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T12:14:28,147 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T12:14:28,148 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T12:14:28,148 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T12:14:28,148 DEBUG [RS:0;843d83d51ba0:42825 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,148 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,42825,1732104864746' 2024-11-20T12:14:28,148 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T12:14:28,149 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T12:14:28,150 DEBUG [RS:0;843d83d51ba0:42825 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T12:14:28,150 INFO [RS:0;843d83d51ba0:42825 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T12:14:28,150 INFO [RS:0;843d83d51ba0:42825 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T12:14:28,255 INFO [RS:0;843d83d51ba0:42825 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:14:28,258 INFO [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C42825%2C1732104864746, suffix=, logDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746, archiveDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/oldWALs, maxLogs=32 2024-11-20T12:14:28,274 DEBUG [RS:0;843d83d51ba0:42825 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746/843d83d51ba0%2C42825%2C1732104864746.1732104868260, exclude list is [], retry=0 2024-11-20T12:14:28,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39791,DS-e866e262-95f8-4b3c-ba6f-4b0dc738e012,DISK] 2024-11-20T12:14:28,284 INFO [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746/843d83d51ba0%2C42825%2C1732104864746.1732104868260 2024-11-20T12:14:28,285 DEBUG [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34329:34329)] 2024-11-20T12:14:28,322 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T12:14:28,322 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:14:28,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741833_1009 (size=32) 2024-11-20T12:14:28,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:28,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T12:14:28,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T12:14:28,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:28,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:28,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T12:14:28,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T12:14:28,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:28,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:28,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T12:14:28,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T12:14:28,753 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:28,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:28,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T12:14:28,756 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T12:14:28,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:28,757 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:28,758 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T12:14:28,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740 2024-11-20T12:14:28,760 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740 2024-11-20T12:14:28,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T12:14:28,763 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T12:14:28,764 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:14:28,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T12:14:28,771 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:14:28,772 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63884750, jitterRate=-0.04804304242134094}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:14:28,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732104868735Initializing all the Stores at 1732104868738 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104868738Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104868738Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104868738Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104868738Cleaning up temporary data from old regions at 1732104868763 (+25 ms)Region opened successfully at 1732104868775 (+12 ms) 2024-11-20T12:14:28,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T12:14:28,775 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T12:14:28,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T12:14:28,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T12:14:28,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T12:14:28,777 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T12:14:28,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732104868775Disabling compacts and flushes for region at 1732104868775Disabling writes for close at 1732104868775Writing region close event to WAL at 1732104868776 (+1 ms)Closed at 1732104868777 (+1 ms) 2024-11-20T12:14:28,781 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:14:28,781 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T12:14:28,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T12:14:28,800 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:14:28,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T12:14:28,957 DEBUG [843d83d51ba0:40015 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T12:14:28,970 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:28,976 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,42825,1732104864746, state=OPENING 2024-11-20T12:14:29,027 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T12:14:29,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:29,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:14:29,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:14:29,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:14:29,200 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:14:29,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:14:29,382 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T12:14:29,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45565, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T12:14:29,396 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T12:14:29,397 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:14:29,397 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T12:14:29,401 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C42825%2C1732104864746.meta, suffix=.meta, logDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746, archiveDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/oldWALs, maxLogs=32 2024-11-20T12:14:29,418 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746/843d83d51ba0%2C42825%2C1732104864746.meta.1732104869404.meta, exclude list is [], retry=0 2024-11-20T12:14:29,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39791,DS-e866e262-95f8-4b3c-ba6f-4b0dc738e012,DISK] 2024-11-20T12:14:29,424 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/WALs/843d83d51ba0,42825,1732104864746/843d83d51ba0%2C42825%2C1732104864746.meta.1732104869404.meta 2024-11-20T12:14:29,425 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34329:34329)] 2024-11-20T12:14:29,425 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:14:29,427 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T12:14:29,429 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T12:14:29,433 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T12:14:29,437 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T12:14:29,437 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:29,438 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T12:14:29,438 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T12:14:29,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T12:14:29,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T12:14:29,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:29,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:29,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T12:14:29,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T12:14:29,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:29,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:29,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T12:14:29,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T12:14:29,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:29,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:29,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T12:14:29,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T12:14:29,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:29,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:14:29,451 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T12:14:29,453 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740 2024-11-20T12:14:29,455 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740 2024-11-20T12:14:29,458 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T12:14:29,458 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T12:14:29,459 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:14:29,462 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T12:14:29,465 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62300102, jitterRate=-0.07165613770484924}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:14:29,465 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T12:14:29,467 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732104869438Writing region info on filesystem at 1732104869438Initializing all the Stores at 1732104869440 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104869440Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104869441 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104869441Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104869441Cleaning up temporary data from old regions at 1732104869458 (+17 ms)Running coprocessor post-open hooks at 1732104869465 (+7 ms)Region opened successfully at 1732104869467 (+2 ms) 2024-11-20T12:14:29,474 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732104869373 2024-11-20T12:14:29,483 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T12:14:29,483 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T12:14:29,485 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:29,487 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,42825,1732104864746, state=OPEN 2024-11-20T12:14:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:14:29,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:14:29,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:14:29,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:14:29,548 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:29,558 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T12:14:29,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,42825,1732104864746 in 347 msec 2024-11-20T12:14:29,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T12:14:29,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 773 msec 2024-11-20T12:14:29,567 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:14:29,567 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T12:14:29,592 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:29,594 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:29,612 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:29,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41665, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:29,633 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8230 sec 2024-11-20T12:14:29,633 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732104869633, completionTime=-1 2024-11-20T12:14:29,636 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T12:14:29,636 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:14:29,659 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-20T12:14:29,659 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104929659 2024-11-20T12:14:29,659 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104989659 2024-11-20T12:14:29,659 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-20T12:14:29,661 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,661 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,662 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,663 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-843d83d51ba0:40015, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,663 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,664 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,670 DEBUG [master/843d83d51ba0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T12:14:29,691 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.592sec 2024-11-20T12:14:29,692 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T12:14:29,693 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T12:14:29,694 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T12:14:29,695 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T12:14:29,695 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T12:14:29,696 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:14:29,697 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T12:14:29,706 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T12:14:29,706 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T12:14:29,707 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,40015,1732104863721-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:14:29,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e129cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:29,798 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T12:14:29,799 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T12:14:29,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:29,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:29,810 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db5a53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:29,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:29,820 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:29,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ebea588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:29,823 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:29,832 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:29,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=843d83d51ba0,40015,1732104863721 2024-11-20T12:14:29,864 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=432, ProcessCount=11, AvailableMemoryMB=1703 2024-11-20T12:14:29,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:14:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:14:29,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f523d87 2024-11-20T12:14:29,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:14:29,908 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:14:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:14:29,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:14:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:29,931 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:14:29,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T12:14:29,933 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:29,936 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:14:29,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:29,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741835_1011 (size=1061) 2024-11-20T12:14:30,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:30,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:30,357 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:14:30,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741836_1012 (size=53) 2024-11-20T12:14:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 56b7424cfd47be723aa0ea7e4fa06ec9, disabling compactions & flushes 2024-11-20T12:14:30,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. after waiting 0 ms 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:30,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:30,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56b7424cfd47be723aa0ea7e4fa06ec9: Waiting for close lock at 1732104870768Disabling compacts and flushes for region at 1732104870768Disabling writes for close at 1732104870768Writing region close event to WAL at 1732104870768Closed at 1732104870768 2024-11-20T12:14:30,771 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:14:30,775 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732104870771"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104870771"}]},"ts":"1732104870771"} 2024-11-20T12:14:30,781 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:14:30,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:14:30,785 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104870783"}]},"ts":"1732104870783"} 2024-11-20T12:14:30,790 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:14:30,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, ASSIGN}] 2024-11-20T12:14:30,795 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, ASSIGN 2024-11-20T12:14:30,796 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:14:30,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56b7424cfd47be723aa0ea7e4fa06ec9, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:30,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, ASSIGN because future has completed 2024-11-20T12:14:30,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:14:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:31,114 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:31,115 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:14:31,115 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,115 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:31,115 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,116 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,118 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,122 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:31,122 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56b7424cfd47be723aa0ea7e4fa06ec9 columnFamilyName A 2024-11-20T12:14:31,122 DEBUG [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:31,124 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(327): Store=56b7424cfd47be723aa0ea7e4fa06ec9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:31,124 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,127 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:31,127 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56b7424cfd47be723aa0ea7e4fa06ec9 columnFamilyName B 2024-11-20T12:14:31,127 DEBUG [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:31,128 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(327): Store=56b7424cfd47be723aa0ea7e4fa06ec9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:31,129 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,131 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:31,131 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56b7424cfd47be723aa0ea7e4fa06ec9 columnFamilyName C 2024-11-20T12:14:31,132 DEBUG [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:31,133 INFO [StoreOpener-56b7424cfd47be723aa0ea7e4fa06ec9-1 {}] regionserver.HStore(327): Store=56b7424cfd47be723aa0ea7e4fa06ec9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:31,133 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:31,134 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,135 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,136 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,139 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:14:31,142 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:14:31,146 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 56b7424cfd47be723aa0ea7e4fa06ec9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70087970, jitterRate=0.044392138719558716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:14:31,146 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:31,147 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 56b7424cfd47be723aa0ea7e4fa06ec9: Running coprocessor pre-open hook at 1732104871116Writing region info on filesystem at 1732104871116Initializing all the Stores at 1732104871118 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104871118Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104871118Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104871118Cleaning up temporary data from old regions at 1732104871138 (+20 ms)Running coprocessor post-open hooks at 1732104871146 (+8 ms)Region opened successfully at 1732104871147 (+1 ms) 2024-11-20T12:14:31,149 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., pid=6, masterSystemTime=1732104871108 2024-11-20T12:14:31,152 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:31,153 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:31,154 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56b7424cfd47be723aa0ea7e4fa06ec9, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:31,159 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:14:31,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T12:14:31,170 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 in 209 msec 2024-11-20T12:14:31,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T12:14:31,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, ASSIGN in 376 msec 2024-11-20T12:14:31,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:14:31,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104871176"}]},"ts":"1732104871176"} 2024-11-20T12:14:31,181 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:14:31,183 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:14:31,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2610 sec 2024-11-20T12:14:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:14:32,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:32,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b3430e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,099 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cdc6cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,102 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1806c24b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,104 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,108 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:14:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:14:32,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78210a9b 2024-11-20T12:14:32,115 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:14:32,117 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:14:32,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7685e9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,123 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,124 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,126 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e3eb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,130 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,134 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b4d8404, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,141 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382062a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,145 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f5332d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,151 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fba9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,154 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5904dd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ef992a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,157 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,157 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b036a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,162 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6675595, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19215e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,166 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20584f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,166 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,168 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7aa9af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64696a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,171 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c287047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,173 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,174 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:14:32,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:14:32,176 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:14:32,177 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:14:32,178 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:32,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:14:32,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:14:32,183 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,183 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,184 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,186 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,186 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T12:14:32,196 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T12:14:32,199 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:32,202 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,202 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,203 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:32,205 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,206 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,208 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,208 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,208 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,213 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:14:32,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,215 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:14:32,217 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:14:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,221 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:14:32,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,227 DEBUG [Thread-154 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:14:32,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:14:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:32,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:32,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:32,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:32,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:32,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:32,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T12:14:32,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T12:14:32,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:32,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104932419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0a848ecbb4274652a22286e2bcb86ea0 is 50, key is test_row_0/A:col10/1732104872268/Put/seqid=0 2024-11-20T12:14:32,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732104872405, max timestamp=1732104872437Num uniques 859; 2024-11-20T12:14:32,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T12:14:32,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104872429, max timestamp=1732104872437Num uniques 859; 2024-11-20T12:14:32,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:14:32,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104872429, max timestamp=1732104872437Num uniques 859; 2024-11-20T12:14:32,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:14:32,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741837_1013 (size=12001) 2024-11-20T12:14:32,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0a848ecbb4274652a22286e2bcb86ea0 2024-11-20T12:14:32,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T12:14:32,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T12:14:32,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d0676124246245ddacd30a24e7d2bc8c is 50, key is test_row_0/B:col10/1732104872268/Put/seqid=0 2024-11-20T12:14:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104932625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741838_1014 (size=12001) 2024-11-20T12:14:32,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104932643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d0676124246245ddacd30a24e7d2bc8c 2024-11-20T12:14:32,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104932643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104932657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104932659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f52ee3fc891d484c8237b3abce0ba4c8 is 50, key is test_row_0/C:col10/1732104872268/Put/seqid=0 2024-11-20T12:14:32,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T12:14:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741839_1015 (size=12001) 2024-11-20T12:14:32,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f52ee3fc891d484c8237b3abce0ba4c8 2024-11-20T12:14:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:32,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:32,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104932768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0a848ecbb4274652a22286e2bcb86ea0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0 2024-11-20T12:14:32,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104932778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104932779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104932779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:32,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T12:14:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104932789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:32,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d0676124246245ddacd30a24e7d2bc8c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c 2024-11-20T12:14:32,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T12:14:32,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f52ee3fc891d484c8237b3abce0ba4c8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8 2024-11-20T12:14:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T12:14:32,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T12:14:32,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 564ms, sequenceid=13, compaction requested=false 2024-11-20T12:14:32,854 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T12:14:32,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:32,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T12:14:32,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:32,927 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T12:14:32,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:32,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:32,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:32,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/637a0aeaac4547b791673cb6a2413a7d is 50, key is test_row_0/A:col10/1732104872625/Put/seqid=0 2024-11-20T12:14:32,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741840_1016 (size=12001) 2024-11-20T12:14:32,994 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/637a0aeaac4547b791673cb6a2413a7d 2024-11-20T12:14:33,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:33,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/5d221cde3b514b0aa51c7ba5a00a7880 is 50, key is test_row_0/B:col10/1732104872625/Put/seqid=0 2024-11-20T12:14:33,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741841_1017 (size=12001) 2024-11-20T12:14:33,052 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/5d221cde3b514b0aa51c7ba5a00a7880 2024-11-20T12:14:33,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104933027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104933031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104933027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104933047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104933053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/abae925f659c44f38e7b6998bb6515e1 is 50, key is test_row_0/C:col10/1732104872625/Put/seqid=0 2024-11-20T12:14:33,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741842_1018 (size=12001) 2024-11-20T12:14:33,107 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/abae925f659c44f38e7b6998bb6515e1 2024-11-20T12:14:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/637a0aeaac4547b791673cb6a2413a7d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d 2024-11-20T12:14:33,148 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T12:14:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/5d221cde3b514b0aa51c7ba5a00a7880 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880 2024-11-20T12:14:33,171 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T12:14:33,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/abae925f659c44f38e7b6998bb6515e1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1 2024-11-20T12:14:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104933160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104933169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104933167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104933169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104933169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,194 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T12:14:33,197 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 270ms, sequenceid=48, compaction requested=false 2024-11-20T12:14:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T12:14:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T12:14:33,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T12:14:33,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-20T12:14:33,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.0210 sec 2024-11-20T12:14:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T12:14:33,337 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:33,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:33,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T12:14:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:33,342 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:33,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:33,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:33,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:14:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:33,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:33,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b12e3dcf52914663968f229bb14fba8e is 50, key is test_row_0/A:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:33,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741843_1019 (size=21365) 2024-11-20T12:14:33,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:33,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b12e3dcf52914663968f229bb14fba8e 2024-11-20T12:14:33,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/76ef1f9df42a4f2e9b018ceeddab0e05 is 50, key is test_row_0/B:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:33,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104933459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104933462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104933478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104933486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104933488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T12:14:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741844_1020 (size=12001) 2024-11-20T12:14:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104933597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104933598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104933598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104933609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104933609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:33,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T12:14:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:33,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104933808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104933808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T12:14:33,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104933817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104933817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104933818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:33,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/76ef1f9df42a4f2e9b018ceeddab0e05 2024-11-20T12:14:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:33,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b63b775636cf470e9e4fdda5f551bce4 is 50, key is test_row_0/C:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:33,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T12:14:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:33,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:33,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:34,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741845_1021 (size=12001) 2024-11-20T12:14:34,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b63b775636cf470e9e4fdda5f551bce4 2024-11-20T12:14:34,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b12e3dcf52914663968f229bb14fba8e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e 2024-11-20T12:14:34,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e, entries=350, sequenceid=63, filesize=20.9 K 2024-11-20T12:14:34,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/76ef1f9df42a4f2e9b018ceeddab0e05 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05 2024-11-20T12:14:34,062 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T12:14:34,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05, entries=150, sequenceid=63, filesize=11.7 K 2024-11-20T12:14:34,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b63b775636cf470e9e4fdda5f551bce4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4 2024-11-20T12:14:34,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4, entries=150, sequenceid=63, filesize=11.7 K 2024-11-20T12:14:34,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 709ms, sequenceid=63, compaction requested=true 2024-11-20T12:14:34,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:34,113 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T12:14:34,113 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T12:14:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:34,133 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:34,135 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:34,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T12:14:34,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:34,137 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:14:34,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:34,139 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:34,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:34,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:34,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:34,142 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:34,143 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:34,144 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:34,145 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:34,145 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:34,145 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:34,145 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.2 K 2024-11-20T12:14:34,145 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=44.3 K 2024-11-20T12:14:34,147 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d0676124246245ddacd30a24e7d2bc8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104872268 2024-11-20T12:14:34,149 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d221cde3b514b0aa51c7ba5a00a7880, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104872613 2024-11-20T12:14:34,150 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ef1f9df42a4f2e9b018ceeddab0e05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873392 2024-11-20T12:14:34,150 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a848ecbb4274652a22286e2bcb86ea0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104872268 2024-11-20T12:14:34,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/9007224ab4b54e7db1510fd25c82b3e0 is 50, key is test_row_0/A:col10/1732104873478/Put/seqid=0 2024-11-20T12:14:34,165 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 637a0aeaac4547b791673cb6a2413a7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104872613 2024-11-20T12:14:34,166 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b12e3dcf52914663968f229bb14fba8e, keycount=350, bloomtype=ROW, size=20.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873018 2024-11-20T12:14:34,178 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T12:14:34,178 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T12:14:34,185 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T12:14:34,185 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T12:14:34,187 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T12:14:34,187 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T12:14:34,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741846_1022 (size=12001) 2024-11-20T12:14:34,214 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/9007224ab4b54e7db1510fd25c82b3e0 2024-11-20T12:14:34,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104934194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4773) at org.apache.hadoop.hbase.regionserver.HRegion.batchMutate(HRegion.java:4699) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5194) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5188) at org.apache.hadoop.hbase.regionserver.HRegion.mutate(HRegion.java:5184) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3333) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,216 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,235 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#11 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104934206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,237 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f8243af4032448deb20269eb9538b747 is 50, key is test_row_0/B:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104934208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,239 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#10 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:34,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104934212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,240 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ea909f4d6d6e429c9e10edacf11f7611 is 50, key is test_row_0/A:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104934216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/abb48da73feb4b7ba23c32c0992e0cbe is 50, key is test_row_0/B:col10/1732104873478/Put/seqid=0 2024-11-20T12:14:34,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741847_1023 (size=12104) 2024-11-20T12:14:34,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741848_1024 (size=12104) 2024-11-20T12:14:34,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741849_1025 (size=12001) 2024-11-20T12:14:34,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104934327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104934348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104934348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104934349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104934349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:34,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104934537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104934567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104934568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104934568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104934569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,677 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f8243af4032448deb20269eb9538b747 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f8243af4032448deb20269eb9538b747 2024-11-20T12:14:34,691 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ea909f4d6d6e429c9e10edacf11f7611 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ea909f4d6d6e429c9e10edacf11f7611 2024-11-20T12:14:34,697 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/abb48da73feb4b7ba23c32c0992e0cbe 2024-11-20T12:14:34,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/4c6b33a4756142dd98b7ef4914ec81de is 50, key is test_row_0/C:col10/1732104873478/Put/seqid=0 2024-11-20T12:14:34,738 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into ea909f4d6d6e429c9e10edacf11f7611(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:34,739 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:34,741 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104874111; duration=0sec 2024-11-20T12:14:34,742 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into f8243af4032448deb20269eb9538b747(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:34,742 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:34,742 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104874138; duration=0sec 2024-11-20T12:14:34,742 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:34,742 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:34,742 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:34,743 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:34,743 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:34,750 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:34,750 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:34,750 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:34,751 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.2 K 2024-11-20T12:14:34,752 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f52ee3fc891d484c8237b3abce0ba4c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104872268 2024-11-20T12:14:34,753 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting abae925f659c44f38e7b6998bb6515e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104872613 2024-11-20T12:14:34,754 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63b775636cf470e9e4fdda5f551bce4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873392 2024-11-20T12:14:34,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741850_1026 (size=12001) 2024-11-20T12:14:34,800 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:34,801 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/fb6ec65e26494540a37b5985235b45e0 is 50, key is test_row_0/C:col10/1732104873393/Put/seqid=0 2024-11-20T12:14:34,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741851_1027 (size=12104) 2024-11-20T12:14:34,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104934848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/fb6ec65e26494540a37b5985235b45e0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fb6ec65e26494540a37b5985235b45e0 2024-11-20T12:14:34,881 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into fb6ec65e26494540a37b5985235b45e0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:34,881 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:34,881 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104874139; duration=0sec 2024-11-20T12:14:34,882 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:34,882 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:34,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104934877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104934889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104934889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104934891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:34,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:14:35,164 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/4c6b33a4756142dd98b7ef4914ec81de 2024-11-20T12:14:35,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/9007224ab4b54e7db1510fd25c82b3e0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0 2024-11-20T12:14:35,192 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0, entries=150, sequenceid=83, filesize=11.7 K 2024-11-20T12:14:35,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/abb48da73feb4b7ba23c32c0992e0cbe as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe 2024-11-20T12:14:35,208 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe, entries=150, sequenceid=83, filesize=11.7 K 2024-11-20T12:14:35,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/4c6b33a4756142dd98b7ef4914ec81de as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de 2024-11-20T12:14:35,230 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de, entries=150, sequenceid=83, filesize=11.7 K 2024-11-20T12:14:35,232 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1095ms, sequenceid=83, compaction requested=false 2024-11-20T12:14:35,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T12:14:35,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T12:14:35,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T12:14:35,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8910 sec 2024-11-20T12:14:35,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9050 sec 2024-11-20T12:14:35,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:14:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:35,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:35,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:35,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/729bea8e70c74e64b7c7d38bc0a4deb7 is 50, key is test_row_0/A:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741852_1028 (size=12001) 2024-11-20T12:14:35,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/729bea8e70c74e64b7c7d38bc0a4deb7 2024-11-20T12:14:35,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e34f8529814f5a97d054736230541e is 50, key is test_row_0/B:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741853_1029 (size=12001) 2024-11-20T12:14:35,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e34f8529814f5a97d054736230541e 2024-11-20T12:14:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T12:14:35,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:35,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:35,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/6a3fad21d7224bcfad5871d5bc1b3397 is 50, key is test_row_0/C:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T12:14:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:35,495 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:35,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:35,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:35,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741854_1030 (size=12001) 2024-11-20T12:14:35,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/6a3fad21d7224bcfad5871d5bc1b3397 2024-11-20T12:14:35,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732104875435, max timestamp=1732104875508Num uniques 859; 2024-11-20T12:14:35,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:14:35,529 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104875435, max timestamp=1732104875508Num uniques 859; 2024-11-20T12:14:35,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:14:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104935527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/729bea8e70c74e64b7c7d38bc0a4deb7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7 2024-11-20T12:14:35,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104875435, max timestamp=1732104875508Num uniques 859; 2024-11-20T12:14:35,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:14:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104935513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T12:14:35,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e34f8529814f5a97d054736230541e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e 2024-11-20T12:14:35,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T12:14:35,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/6a3fad21d7224bcfad5871d5bc1b3397 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397 2024-11-20T12:14:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:35,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397, entries=150, sequenceid=99, filesize=11.7 K 2024-11-20T12:14:35,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=123.00 KB/125950 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 255ms, sequenceid=99, compaction requested=true 2024-11-20T12:14:35,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:35,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:35,624 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:35,626 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:35,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:35,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-20T12:14:35,627 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:35,627 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:35,627 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,628 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ea909f4d6d6e429c9e10edacf11f7611, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.3 K 2024-11-20T12:14:35,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:35,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:35,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:35,629 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea909f4d6d6e429c9e10edacf11f7611, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873392 2024-11-20T12:14:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:35,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:35,629 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:35,629 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:35,629 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,629 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f8243af4032448deb20269eb9538b747, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.3 K 2024-11-20T12:14:35,630 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9007224ab4b54e7db1510fd25c82b3e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732104873478 2024-11-20T12:14:35,630 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f8243af4032448deb20269eb9538b747, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873392 2024-11-20T12:14:35,631 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 729bea8e70c74e64b7c7d38bc0a4deb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:35,636 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting abb48da73feb4b7ba23c32c0992e0cbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732104873478 2024-11-20T12:14:35,637 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e34f8529814f5a97d054736230541e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:35,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:35,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/6c875641dbe64d9eac7822925de5b7c7 is 50, key is test_row_0/A:col10/1732104875606/Put/seqid=0 2024-11-20T12:14:35,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,668 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:35,670 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/1ec865a2b1ba47349730108b999f0fb9 is 50, key is test_row_0/A:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104935657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104935657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104935673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,681 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:35,682 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/5f0ee78a43cd45e8818a286dbecca72c is 50, key is test_row_0/B:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104935677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104935677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741856_1032 (size=12207) 2024-11-20T12:14:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,718 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/1ec865a2b1ba47349730108b999f0fb9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/1ec865a2b1ba47349730108b999f0fb9 2024-11-20T12:14:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741855_1031 (size=21615) 2024-11-20T12:14:35,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/6c875641dbe64d9eac7822925de5b7c7 2024-11-20T12:14:35,736 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into 1ec865a2b1ba47349730108b999f0fb9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:35,736 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:35,736 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104875624; duration=0sec 2024-11-20T12:14:35,737 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:35,737 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:35,737 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:35,740 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:35,740 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:35,740 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,740 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fb6ec65e26494540a37b5985235b45e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.3 K 2024-11-20T12:14:35,743 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb6ec65e26494540a37b5985235b45e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732104873392 2024-11-20T12:14:35,745 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c6b33a4756142dd98b7ef4914ec81de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732104873478 2024-11-20T12:14:35,746 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a3fad21d7224bcfad5871d5bc1b3397, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:35,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741857_1033 (size=12207) 2024-11-20T12:14:35,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/54af9999452540eea77c3c11f4900f3b is 50, key is test_row_0/B:col10/1732104875606/Put/seqid=0 2024-11-20T12:14:35,770 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/5f0ee78a43cd45e8818a286dbecca72c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5f0ee78a43cd45e8818a286dbecca72c 2024-11-20T12:14:35,782 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into 5f0ee78a43cd45e8818a286dbecca72c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:35,782 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:35,783 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104875626; duration=0sec 2024-11-20T12:14:35,783 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:35,783 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:35,784 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:35,785 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c12f581d6061481da3232e3304802bbf is 50, key is test_row_0/C:col10/1732104874207/Put/seqid=0 2024-11-20T12:14:35,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741859_1035 (size=12207) 2024-11-20T12:14:35,811 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c12f581d6061481da3232e3304802bbf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c12f581d6061481da3232e3304802bbf 2024-11-20T12:14:35,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104935807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104935807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741858_1034 (size=12151) 2024-11-20T12:14:35,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/54af9999452540eea77c3c11f4900f3b 2024-11-20T12:14:35,826 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into c12f581d6061481da3232e3304802bbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:35,826 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:35,826 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104875626; duration=0sec 2024-11-20T12:14:35,826 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:35,826 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:35,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104935838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/beb01876d27a4c7d935f1c1023c71fc6 is 50, key is test_row_0/C:col10/1732104875606/Put/seqid=0 2024-11-20T12:14:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741860_1036 (size=12151) 2024-11-20T12:14:35,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104935906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104935906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:35,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:35,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:35,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104936027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104936027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104936047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:36,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104936217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104936217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/beb01876d27a4c7d935f1c1023c71fc6 2024-11-20T12:14:36,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/6c875641dbe64d9eac7822925de5b7c7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7 2024-11-20T12:14:36,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:36,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7, entries=350, sequenceid=136, filesize=21.1 K 2024-11-20T12:14:36,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/54af9999452540eea77c3c11f4900f3b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b 2024-11-20T12:14:36,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T12:14:36,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/beb01876d27a4c7d935f1c1023c71fc6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6 2024-11-20T12:14:36,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T12:14:36,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 701ms, sequenceid=136, compaction requested=false 2024-11-20T12:14:36,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:36,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:14:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b0c8e90824ec45abbc1fa99088666bde is 50, key is test_row_0/A:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741861_1037 (size=14541) 2024-11-20T12:14:36,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b0c8e90824ec45abbc1fa99088666bde 2024-11-20T12:14:36,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/7d4476ae63ec475a96563b23e8ecdb69 is 50, key is test_row_0/B:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104936393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104936399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104936402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:36,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:36,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:36,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741862_1038 (size=12151) 2024-11-20T12:14:36,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/7d4476ae63ec475a96563b23e8ecdb69 2024-11-20T12:14:36,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/fe1fb3abbd324801b456e0d0f25c1eed is 50, key is test_row_0/C:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741863_1039 (size=12151) 2024-11-20T12:14:36,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/fe1fb3abbd324801b456e0d0f25c1eed 2024-11-20T12:14:36,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104936508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b0c8e90824ec45abbc1fa99088666bde as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde 2024-11-20T12:14:36,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104936517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104936517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde, entries=200, sequenceid=151, filesize=14.2 K 2024-11-20T12:14:36,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/7d4476ae63ec475a96563b23e8ecdb69 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69 2024-11-20T12:14:36,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69, entries=150, sequenceid=151, filesize=11.9 K 2024-11-20T12:14:36,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/fe1fb3abbd324801b456e0d0f25c1eed as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed 2024-11-20T12:14:36,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed, entries=150, sequenceid=151, filesize=11.9 K 2024-11-20T12:14:36,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 216ms, sequenceid=151, compaction requested=true 2024-11-20T12:14:36,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:36,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:36,558 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:36,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:36,559 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:36,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:36,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:36,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:36,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:36,560 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 48363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:36,560 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:36,560 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,561 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/1ec865a2b1ba47349730108b999f0fb9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=47.2 K 2024-11-20T12:14:36,561 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:36,561 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:36,561 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,561 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5f0ee78a43cd45e8818a286dbecca72c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.7 K 2024-11-20T12:14:36,561 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ec865a2b1ba47349730108b999f0fb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:36,562 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c875641dbe64d9eac7822925de5b7c7, keycount=350, bloomtype=ROW, size=21.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104875528 2024-11-20T12:14:36,563 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0ee78a43cd45e8818a286dbecca72c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:36,563 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0c8e90824ec45abbc1fa99088666bde, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875665 2024-11-20T12:14:36,564 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 54af9999452540eea77c3c11f4900f3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104875606 2024-11-20T12:14:36,565 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4476ae63ec475a96563b23e8ecdb69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875672 2024-11-20T12:14:36,589 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#27 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:36,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T12:14:36,590 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3f99ff146dfd48f39fcd7f381229a232 is 50, key is test_row_0/A:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,590 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:14:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:36,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:36,599 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:36,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f4a712dde22f4d3984e0fb5e1cd7d807 is 50, key is test_row_0/B:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c006c93ea3b1407289a8f19aaebfc7e2 is 50, key is test_row_0/A:col10/1732104876398/Put/seqid=0 2024-11-20T12:14:36,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741864_1040 (size=12459) 2024-11-20T12:14:36,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741865_1041 (size=12459) 2024-11-20T12:14:36,629 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3f99ff146dfd48f39fcd7f381229a232 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3f99ff146dfd48f39fcd7f381229a232 2024-11-20T12:14:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:36,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741866_1042 (size=12151) 2024-11-20T12:14:36,645 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c006c93ea3b1407289a8f19aaebfc7e2 2024-11-20T12:14:36,669 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f4a712dde22f4d3984e0fb5e1cd7d807 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4a712dde22f4d3984e0fb5e1cd7d807 2024-11-20T12:14:36,674 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into 3f99ff146dfd48f39fcd7f381229a232(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:36,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:36,674 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104876558; duration=0sec 2024-11-20T12:14:36,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:36,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:36,675 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:36,681 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:36,681 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into f4a712dde22f4d3984e0fb5e1cd7d807(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:36,681 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:36,681 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:36,681 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104876558; duration=0sec 2024-11-20T12:14:36,681 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:36,682 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c12f581d6061481da3232e3304802bbf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.7 K 2024-11-20T12:14:36,682 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:36,682 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:36,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/b1264092ee0f4b75b85f12ef4cf74db7 is 50, key is test_row_0/B:col10/1732104876398/Put/seqid=0 2024-11-20T12:14:36,683 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c12f581d6061481da3232e3304802bbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732104874207 2024-11-20T12:14:36,684 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting beb01876d27a4c7d935f1c1023c71fc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104875606 2024-11-20T12:14:36,685 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe1fb3abbd324801b456e0d0f25c1eed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875672 2024-11-20T12:14:36,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741867_1043 (size=12151) 2024-11-20T12:14:36,715 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:36,715 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/16539d5976554be9b3a37cd90d640897 is 50, key is test_row_0/C:col10/1732104876339/Put/seqid=0 2024-11-20T12:14:36,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:36,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741868_1044 (size=12459) 2024-11-20T12:14:36,747 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/16539d5976554be9b3a37cd90d640897 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/16539d5976554be9b3a37cd90d640897 2024-11-20T12:14:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104936743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104936745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104936746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,760 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into 16539d5976554be9b3a37cd90d640897(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:36,760 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,760 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104876559; duration=0sec 2024-11-20T12:14:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104936746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,761 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:36,761 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104936748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104936866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104936867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104936867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104936867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104936867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104937078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104937079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104937079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104937079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104937079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,113 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/b1264092ee0f4b75b85f12ef4cf74db7 2024-11-20T12:14:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/cc8405197f1d43b88daa8d3fbb471164 is 50, key is test_row_0/C:col10/1732104876398/Put/seqid=0 2024-11-20T12:14:37,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741869_1045 (size=12151) 2024-11-20T12:14:37,156 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/cc8405197f1d43b88daa8d3fbb471164 2024-11-20T12:14:37,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c006c93ea3b1407289a8f19aaebfc7e2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2 2024-11-20T12:14:37,180 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T12:14:37,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/b1264092ee0f4b75b85f12ef4cf74db7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7 2024-11-20T12:14:37,201 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T12:14:37,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/cc8405197f1d43b88daa8d3fbb471164 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164 2024-11-20T12:14:37,221 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T12:14:37,225 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 635ms, sequenceid=171, compaction requested=false 2024-11-20T12:14:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:37,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T12:14:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T12:14:37,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T12:14:37,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7310 sec 2024-11-20T12:14:37,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7470 sec 2024-11-20T12:14:37,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:37,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:37,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/cf5d4d7e2dac4653a9c93b16ecec5a5d is 50, key is test_row_0/A:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:37,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104937418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104937421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104937427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104937429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104937437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741870_1046 (size=14541) 2024-11-20T12:14:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/cf5d4d7e2dac4653a9c93b16ecec5a5d 2024-11-20T12:14:37,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e7aec9f728411da8308d0f7bffc399 is 50, key is test_row_0/B:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:37,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741871_1047 (size=12151) 2024-11-20T12:14:37,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e7aec9f728411da8308d0f7bffc399 2024-11-20T12:14:37,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/ae8b564adb9944acb0d55f27184c4040 is 50, key is test_row_0/C:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104937538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741872_1048 (size=12151) 2024-11-20T12:14:37,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104937547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104937547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104937557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104937547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T12:14:37,648 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:37,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T12:14:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:37,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:37,655 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:37,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:37,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104937747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:37,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104937767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104937767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104937767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:37,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104937767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:37,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:37,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:37,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:37,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/ae8b564adb9944acb0d55f27184c4040 2024-11-20T12:14:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:37,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/cf5d4d7e2dac4653a9c93b16ecec5a5d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d 2024-11-20T12:14:37,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d, entries=200, sequenceid=190, filesize=14.2 K 2024-11-20T12:14:37,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e3e7aec9f728411da8308d0f7bffc399 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399 2024-11-20T12:14:37,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:37,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:37,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:37,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:37,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:37,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T12:14:37,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/ae8b564adb9944acb0d55f27184c4040 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040 2024-11-20T12:14:38,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T12:14:38,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 621ms, sequenceid=190, compaction requested=true 2024-11-20T12:14:38,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:38,013 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:38,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:38,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:38,014 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:38,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:38,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:38,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:38,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:38,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:38,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:38,016 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,017 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3f99ff146dfd48f39fcd7f381229a232, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=38.2 K 2024-11-20T12:14:38,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:38,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:38,017 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,017 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4a712dde22f4d3984e0fb5e1cd7d807, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.9 K 2024-11-20T12:14:38,018 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f99ff146dfd48f39fcd7f381229a232, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875672 2024-11-20T12:14:38,018 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f4a712dde22f4d3984e0fb5e1cd7d807, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875672 2024-11-20T12:14:38,019 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c006c93ea3b1407289a8f19aaebfc7e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732104876370 2024-11-20T12:14:38,021 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5d4d7e2dac4653a9c93b16ecec5a5d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:38,022 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b1264092ee0f4b75b85f12ef4cf74db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732104876370 2024-11-20T12:14:38,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e7aec9f728411da8308d0f7bffc399, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:38,048 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:38,049 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b6e82b4d5f3e4664801fdfc69483ada7 is 50, key is test_row_0/A:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:38,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:14:38,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:38,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:38,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:38,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:38,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:38,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:38,067 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:38,068 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/b717d7de182a42c98bb4716600f4f590 is 50, key is test_row_0/B:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:38,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2e74fe761bd74e85a3ee6f28c66e508b is 50, key is test_row_0/A:col10/1732104877403/Put/seqid=0 2024-11-20T12:14:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741873_1049 (size=12561) 2024-11-20T12:14:38,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741874_1050 (size=12561) 2024-11-20T12:14:38,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104938127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104938127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104938131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104938135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104938143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,152 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/b717d7de182a42c98bb4716600f4f590 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b717d7de182a42c98bb4716600f4f590 2024-11-20T12:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,166 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into b717d7de182a42c98bb4716600f4f590(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:38,167 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:38,167 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104878014; duration=0sec 2024-11-20T12:14:38,167 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:38,167 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:38,167 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:38,169 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:38,169 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:38,169 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,170 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/16539d5976554be9b3a37cd90d640897, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=35.9 K 2024-11-20T12:14:38,170 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 16539d5976554be9b3a37cd90d640897, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732104875672 2024-11-20T12:14:38,171 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8405197f1d43b88daa8d3fbb471164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732104876370 2024-11-20T12:14:38,171 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ae8b564adb9944acb0d55f27184c4040, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:38,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741875_1051 (size=14541) 2024-11-20T12:14:38,207 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:38,210 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/df4cdb3aeb424566baf0c611e766e666 is 50, key is test_row_0/C:col10/1732104877391/Put/seqid=0 2024-11-20T12:14:38,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741876_1052 (size=12561) 2024-11-20T12:14:38,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104938257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104938258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104938258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104938258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104938258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,270 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/df4cdb3aeb424566baf0c611e766e666 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/df4cdb3aeb424566baf0c611e766e666 2024-11-20T12:14:38,283 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into df4cdb3aeb424566baf0c611e766e666(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:38,283 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:38,283 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104878014; duration=0sec 2024-11-20T12:14:38,283 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:38,283 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:38,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104938466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104938467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104938469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104938469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104938472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,532 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b6e82b4d5f3e4664801fdfc69483ada7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b6e82b4d5f3e4664801fdfc69483ada7 2024-11-20T12:14:38,567 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into b6e82b4d5f3e4664801fdfc69483ada7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:38,567 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:38,567 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104878013; duration=0sec 2024-11-20T12:14:38,567 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:38,567 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:38,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2e74fe761bd74e85a3ee6f28c66e508b 2024-11-20T12:14:38,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4a6ada9808ab44989585e56cf5142673 is 50, key is test_row_0/B:col10/1732104877403/Put/seqid=0 2024-11-20T12:14:38,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741877_1053 (size=12151) 2024-11-20T12:14:38,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104938787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104938787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104938788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:38,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104938788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104938788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:38,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:38,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:38,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:38,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4a6ada9808ab44989585e56cf5142673 2024-11-20T12:14:39,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/2123803a8a9f4f5fb39f53b5dfb279cf is 50, key is test_row_0/C:col10/1732104877403/Put/seqid=0 2024-11-20T12:14:39,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741878_1054 (size=12151) 2024-11-20T12:14:39,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104939298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:39,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:39,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104939307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104939307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104939312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104939308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:39,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:39,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:39,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/2123803a8a9f4f5fb39f53b5dfb279cf 2024-11-20T12:14:39,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2e74fe761bd74e85a3ee6f28c66e508b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b 2024-11-20T12:14:39,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b, entries=200, sequenceid=207, filesize=14.2 K 2024-11-20T12:14:39,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4a6ada9808ab44989585e56cf5142673 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673 2024-11-20T12:14:39,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T12:14:39,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/2123803a8a9f4f5fb39f53b5dfb279cf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf 2024-11-20T12:14:39,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T12:14:39,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1458ms, sequenceid=207, compaction requested=false 2024-11-20T12:14:39,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:39,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T12:14:39,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:39,536 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T12:14:39,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:39,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b70826f63b814f6fab4b49fafbb62116 is 50, key is test_row_0/A:col10/1732104878134/Put/seqid=0 2024-11-20T12:14:39,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741879_1055 (size=12151) 2024-11-20T12:14:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:39,961 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b70826f63b814f6fab4b49fafbb62116 2024-11-20T12:14:39,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/03f481d6270d4d6fa121753e4f560de4 is 50, key is test_row_0/B:col10/1732104878134/Put/seqid=0 2024-11-20T12:14:39,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741880_1056 (size=12151) 2024-11-20T12:14:39,990 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/03f481d6270d4d6fa121753e4f560de4 2024-11-20T12:14:40,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bd00e533040c465cba506e78dde585f4 is 50, key is test_row_0/C:col10/1732104878134/Put/seqid=0 2024-11-20T12:14:40,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741881_1057 (size=12151) 2024-11-20T12:14:40,048 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bd00e533040c465cba506e78dde585f4 2024-11-20T12:14:40,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/b70826f63b814f6fab4b49fafbb62116 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116 2024-11-20T12:14:40,065 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:14:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/03f481d6270d4d6fa121753e4f560de4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4 2024-11-20T12:14:40,081 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:14:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bd00e533040c465cba506e78dde585f4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4 2024-11-20T12:14:40,100 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:14:40,102 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 566ms, sequenceid=225, compaction requested=true 2024-11-20T12:14:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T12:14:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T12:14:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T12:14:40,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4490 sec 2024-11-20T12:14:40,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.4710 sec 2024-11-20T12:14:40,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:14:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:40,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0e4c86694a31450abb14080b3fa61d1c is 50, key is test_row_0/A:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:40,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741882_1058 (size=12151) 2024-11-20T12:14:40,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104880363, max timestamp=1732104880378Num uniques 859; 2024-11-20T12:14:40,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:14:40,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104880363, max timestamp=1732104880378Num uniques 859; 2024-11-20T12:14:40,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:14:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104880363, max timestamp=1732104880378Num uniques 859; 2024-11-20T12:14:40,389 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:14:40,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104940454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104940454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104940456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104940461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104940463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104940567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104940567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104940567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104940576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104940576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0e4c86694a31450abb14080b3fa61d1c 2024-11-20T12:14:40,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f57c9b47034e4322a0127c6c33ba794d is 50, key is test_row_0/B:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:40,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741883_1059 (size=12151) 2024-11-20T12:14:40,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104940776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104940777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,784 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104940777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104940787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104940787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:40,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104941087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104941097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104941098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104941098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104941098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f57c9b47034e4322a0127c6c33ba794d 2024-11-20T12:14:41,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f3cb1b83d7b14f5aa0f468adc8b377ed is 50, key is test_row_0/C:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:41,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741884_1060 (size=12151) 2024-11-20T12:14:41,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104941607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104941607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104941607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104941607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104941608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:41,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f3cb1b83d7b14f5aa0f468adc8b377ed 2024-11-20T12:14:41,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/0e4c86694a31450abb14080b3fa61d1c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c 2024-11-20T12:14:41,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:14:41,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f57c9b47034e4322a0127c6c33ba794d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d 2024-11-20T12:14:41,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:14:41,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/f3cb1b83d7b14f5aa0f468adc8b377ed as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed 2024-11-20T12:14:41,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:14:41,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1323ms, sequenceid=236, compaction requested=true 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:41,658 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:41,658 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:41,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:41,659 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:41,660 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:41,660 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:41,660 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:41,660 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:41,660 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:41,660 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b6e82b4d5f3e4664801fdfc69483ada7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=50.2 K 2024-11-20T12:14:41,660 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b717d7de182a42c98bb4716600f4f590, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=47.9 K 2024-11-20T12:14:41,660 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e82b4d5f3e4664801fdfc69483ada7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:41,660 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b717d7de182a42c98bb4716600f4f590, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e74fe761bd74e85a3ee6f28c66e508b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732104877403 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a6ada9808ab44989585e56cf5142673, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732104877403 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70826f63b814f6fab4b49fafbb62116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104878109 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f481d6270d4d6fa121753e4f560de4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104878109 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4c86694a31450abb14080b3fa61d1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:41,661 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f57c9b47034e4322a0127c6c33ba794d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:41,672 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:41,672 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:41,673 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e9970552882c4ea4a1b673c9ccaa31e9 is 50, key is test_row_0/B:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:41,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/21fa049611834f9596949aee27770dc6 is 50, key is test_row_0/A:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:41,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741885_1061 (size=12697) 2024-11-20T12:14:41,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741886_1062 (size=12697) 2024-11-20T12:14:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T12:14:41,817 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:41,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T12:14:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T12:14:41,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:41,820 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:41,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T12:14:41,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T12:14:41,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:41,973 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:14:41,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2d5d0c20458847218e517b85cc18b94f is 50, key is test_row_0/A:col10/1732104880459/Put/seqid=0 2024-11-20T12:14:41,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741887_1063 (size=12301) 2024-11-20T12:14:42,097 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/e9970552882c4ea4a1b673c9ccaa31e9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e9970552882c4ea4a1b673c9ccaa31e9 2024-11-20T12:14:42,097 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/21fa049611834f9596949aee27770dc6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/21fa049611834f9596949aee27770dc6 2024-11-20T12:14:42,105 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into e9970552882c4ea4a1b673c9ccaa31e9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:42,105 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into 21fa049611834f9596949aee27770dc6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:42,105 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:42,105 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:42,105 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=12, startTime=1732104881658; duration=0sec 2024-11-20T12:14:42,105 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=12, startTime=1732104881658; duration=0sec 2024-11-20T12:14:42,105 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:42,106 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:42,105 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:42,106 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:42,106 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:42,107 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:42,107 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:42,107 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:42,108 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/df4cdb3aeb424566baf0c611e766e666, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=47.9 K 2024-11-20T12:14:42,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting df4cdb3aeb424566baf0c611e766e666, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732104876739 2024-11-20T12:14:42,111 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 2123803a8a9f4f5fb39f53b5dfb279cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732104877403 2024-11-20T12:14:42,112 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting bd00e533040c465cba506e78dde585f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104878109 2024-11-20T12:14:42,112 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f3cb1b83d7b14f5aa0f468adc8b377ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:42,135 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#51 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:42,136 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c4035dfadb2248af8f5c01c922cc3a0f is 50, key is test_row_0/C:col10/1732104880330/Put/seqid=0 2024-11-20T12:14:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T12:14:42,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741888_1064 (size=12697) 2024-11-20T12:14:42,164 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c4035dfadb2248af8f5c01c922cc3a0f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c4035dfadb2248af8f5c01c922cc3a0f 2024-11-20T12:14:42,173 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into c4035dfadb2248af8f5c01c922cc3a0f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:42,173 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:42,173 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=12, startTime=1732104881658; duration=0sec 2024-11-20T12:14:42,173 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:42,173 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:42,385 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2d5d0c20458847218e517b85cc18b94f 2024-11-20T12:14:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d82ad9f81ea54873a915a23dfe632923 is 50, key is test_row_0/B:col10/1732104880459/Put/seqid=0 2024-11-20T12:14:42,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741889_1065 (size=12301) 2024-11-20T12:14:42,411 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d82ad9f81ea54873a915a23dfe632923 2024-11-20T12:14:42,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/00559e1690614d0ab1654d0a5115b940 is 50, key is test_row_0/C:col10/1732104880459/Put/seqid=0 2024-11-20T12:14:42,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T12:14:42,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741890_1066 (size=12301) 2024-11-20T12:14:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:42,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104942638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104942642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104942642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104942642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104942647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104942757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104942757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104942757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104942757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:42,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104942757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:42,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:42,856 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/00559e1690614d0ab1654d0a5115b940 2024-11-20T12:14:42,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/2d5d0c20458847218e517b85cc18b94f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f 2024-11-20T12:14:42,882 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f, entries=150, sequenceid=270, filesize=12.0 K 2024-11-20T12:14:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d82ad9f81ea54873a915a23dfe632923 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923 2024-11-20T12:14:42,909 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923, entries=150, sequenceid=270, filesize=12.0 K 2024-11-20T12:14:42,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/00559e1690614d0ab1654d0a5115b940 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940 2024-11-20T12:14:42,928 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940, entries=150, sequenceid=270, filesize=12.0 K 2024-11-20T12:14:42,930 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 957ms, sequenceid=270, compaction requested=false 2024-11-20T12:14:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:42,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T12:14:42,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T12:14:42,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T12:14:42,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-20T12:14:42,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1190 sec 2024-11-20T12:14:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T12:14:42,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:42,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T12:14:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:42,962 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:42,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:42,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:42,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:42,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:42,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3e1a204b1f6640ef85716af0a3173100 is 50, key is test_row_0/A:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741891_1067 (size=14737) 2024-11-20T12:14:43,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3e1a204b1f6640ef85716af0a3173100 2024-11-20T12:14:43,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/2b587fb702474087bb650a785f7e098f is 50, key is test_row_0/B:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741892_1068 (size=9857) 2024-11-20T12:14:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104943014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104943019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:43,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104943060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104943060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104943060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T12:14:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:43,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104943166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104943167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104943177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104943177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104943186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T12:14:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:43,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104943377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104943377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104943387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104943387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104943401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T12:14:43,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:43,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:43,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/2b587fb702474087bb650a785f7e098f 2024-11-20T12:14:43,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c514179ae94f429196c34b305650d045 is 50, key is test_row_0/C:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741893_1069 (size=9857) 2024-11-20T12:14:43,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c514179ae94f429196c34b305650d045 2024-11-20T12:14:43,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3e1a204b1f6640ef85716af0a3173100 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100 2024-11-20T12:14:43,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100, entries=200, sequenceid=286, filesize=14.4 K 2024-11-20T12:14:43,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/2b587fb702474087bb650a785f7e098f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f 2024-11-20T12:14:43,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f, entries=100, sequenceid=286, filesize=9.6 K 2024-11-20T12:14:43,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/c514179ae94f429196c34b305650d045 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045 2024-11-20T12:14:43,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045, entries=100, sequenceid=286, filesize=9.6 K 2024-11-20T12:14:43,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 574ms, sequenceid=286, compaction requested=true 2024-11-20T12:14:43,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:43,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:43,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:43,545 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:43,545 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:43,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:43,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:43,547 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39735 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:43,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:43,547 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:43,547 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,547 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,547 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e9970552882c4ea4a1b673c9ccaa31e9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=34.0 K 2024-11-20T12:14:43,547 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/21fa049611834f9596949aee27770dc6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=38.8 K 2024-11-20T12:14:43,550 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e9970552882c4ea4a1b673c9ccaa31e9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:43,550 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21fa049611834f9596949aee27770dc6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:43,550 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5d0c20458847218e517b85cc18b94f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732104880404 2024-11-20T12:14:43,550 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d82ad9f81ea54873a915a23dfe632923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732104880404 2024-11-20T12:14:43,551 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e1a204b1f6640ef85716af0a3173100, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104882638 2024-11-20T12:14:43,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b587fb702474087bb650a785f7e098f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104882644 2024-11-20T12:14:43,565 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:43,566 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/8715987d308644edbca242eef38da853 is 50, key is test_row_0/A:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,569 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:43,570 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f69e47f6011a4c3eb0fde27aad027141 is 50, key is test_row_0/B:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741894_1070 (size=12949) 2024-11-20T12:14:43,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741895_1071 (size=12949) 2024-11-20T12:14:43,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T12:14:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,579 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:14:43,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:43,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:43,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:43,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/eec085caab654e9f8e09122206f3ae0f is 50, key is test_row_0/A:col10/1732104883033/Put/seqid=0 2024-11-20T12:14:43,592 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/8715987d308644edbca242eef38da853 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/8715987d308644edbca242eef38da853 2024-11-20T12:14:43,605 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f69e47f6011a4c3eb0fde27aad027141 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f69e47f6011a4c3eb0fde27aad027141 2024-11-20T12:14:43,605 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into 8715987d308644edbca242eef38da853(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:43,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:43,605 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104883545; duration=0sec 2024-11-20T12:14:43,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:43,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:43,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:43,607 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:43,607 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:43,607 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:43,607 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c4035dfadb2248af8f5c01c922cc3a0f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=34.0 K 2024-11-20T12:14:43,608 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4035dfadb2248af8f5c01c922cc3a0f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104880330 2024-11-20T12:14:43,609 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00559e1690614d0ab1654d0a5115b940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1732104880404 2024-11-20T12:14:43,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741896_1072 (size=12301) 2024-11-20T12:14:43,612 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c514179ae94f429196c34b305650d045, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104882644 2024-11-20T12:14:43,614 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into f69e47f6011a4c3eb0fde27aad027141(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:43,614 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:43,614 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104883545; duration=0sec 2024-11-20T12:14:43,615 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:43,615 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:43,621 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:43,622 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/0518a77392b842a193da4391c9ecd41e is 50, key is test_row_0/C:col10/1732104882970/Put/seqid=0 2024-11-20T12:14:43,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741897_1073 (size=12949) 2024-11-20T12:14:43,634 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/0518a77392b842a193da4391c9ecd41e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0518a77392b842a193da4391c9ecd41e 2024-11-20T12:14:43,642 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into 0518a77392b842a193da4391c9ecd41e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:43,642 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:43,642 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104883546; duration=0sec 2024-11-20T12:14:43,642 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:43,643 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:43,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:43,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104943707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104943707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104943707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104943707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104943709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104943817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104943817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104943817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,013 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/eec085caab654e9f8e09122206f3ae0f 2024-11-20T12:14:44,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/c1a92321c14b411e8ba36150f395757f is 50, key is test_row_0/B:col10/1732104883033/Put/seqid=0 2024-11-20T12:14:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104944027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104944027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104944027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741898_1074 (size=12301) 2024-11-20T12:14:44,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:44,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104944220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104944227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104944337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104944337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104944337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,434 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/c1a92321c14b411e8ba36150f395757f 2024-11-20T12:14:44,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b05a137fdf704c7eb96e5c388224694c is 50, key is test_row_0/C:col10/1732104883033/Put/seqid=0 2024-11-20T12:14:44,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741899_1075 (size=12301) 2024-11-20T12:14:44,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104944847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104944847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104944847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:44,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:44,851 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b05a137fdf704c7eb96e5c388224694c 2024-11-20T12:14:44,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/eec085caab654e9f8e09122206f3ae0f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f 2024-11-20T12:14:44,862 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f, entries=150, sequenceid=305, filesize=12.0 K 2024-11-20T12:14:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/c1a92321c14b411e8ba36150f395757f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f 2024-11-20T12:14:44,868 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f, entries=150, sequenceid=305, filesize=12.0 K 2024-11-20T12:14:44,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/b05a137fdf704c7eb96e5c388224694c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c 2024-11-20T12:14:44,875 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c, entries=150, sequenceid=305, filesize=12.0 K 2024-11-20T12:14:44,880 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1301ms, sequenceid=305, compaction requested=false 2024-11-20T12:14:44,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:44,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:44,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T12:14:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T12:14:44,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T12:14:44,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9180 sec 2024-11-20T12:14:44,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.9270 sec 2024-11-20T12:14:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T12:14:45,107 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:45,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T12:14:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T12:14:45,109 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:45,110 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:45,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T12:14:45,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:14:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:45,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bf5004d00055435ab41eafe0e12b50cd is 50, key is test_row_0/A:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T12:14:45,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741900_1076 (size=17181) 2024-11-20T12:14:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bf5004d00055435ab41eafe0e12b50cd 2024-11-20T12:14:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ce50c53ad27a4a5fba0f434d75d441d9 is 50, key is test_row_0/B:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741901_1077 (size=12301) 2024-11-20T12:14:45,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ce50c53ad27a4a5fba0f434d75d441d9 2024-11-20T12:14:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104945292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104945294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/613cf54f194b445d835434cae9643319 is 50, key is test_row_0/C:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741902_1078 (size=12301) 2024-11-20T12:14:45,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/613cf54f194b445d835434cae9643319 2024-11-20T12:14:45,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bf5004d00055435ab41eafe0e12b50cd as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd 2024-11-20T12:14:45,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd, entries=250, sequenceid=323, filesize=16.8 K 2024-11-20T12:14:45,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ce50c53ad27a4a5fba0f434d75d441d9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9 2024-11-20T12:14:45,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T12:14:45,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104945408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/613cf54f194b445d835434cae9643319 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319 2024-11-20T12:14:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104945408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T12:14:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:45,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319, entries=150, sequenceid=323, filesize=12.0 K 2024-11-20T12:14:45,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 188ms, sequenceid=323, compaction requested=true 2024-11-20T12:14:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:45,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:45,428 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:45,429 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:45,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T12:14:45,431 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:45,431 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:45,431 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,431 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f69e47f6011a4c3eb0fde27aad027141, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=36.7 K 2024-11-20T12:14:45,431 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:45,432 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:45,432 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f69e47f6011a4c3eb0fde27aad027141, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104880460 2024-11-20T12:14:45,432 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,432 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/8715987d308644edbca242eef38da853, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=41.4 K 2024-11-20T12:14:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:45,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:45,432 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8715987d308644edbca242eef38da853, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104880460 2024-11-20T12:14:45,432 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a92321c14b411e8ba36150f395757f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732104883017 2024-11-20T12:14:45,433 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting eec085caab654e9f8e09122206f3ae0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732104883017 2024-11-20T12:14:45,433 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ce50c53ad27a4a5fba0f434d75d441d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883705 2024-11-20T12:14:45,434 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5004d00055435ab41eafe0e12b50cd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883699 2024-11-20T12:14:45,451 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:45,451 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/447eac8560d84b92b206d49236a58d49 is 50, key is test_row_0/A:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,459 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:45,460 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/655c4e58c03540fbb60f72c9f647fa29 is 50, key is test_row_0/B:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741903_1079 (size=13051) 2024-11-20T12:14:45,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741904_1080 (size=13051) 2024-11-20T12:14:45,502 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/655c4e58c03540fbb60f72c9f647fa29 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/655c4e58c03540fbb60f72c9f647fa29 2024-11-20T12:14:45,513 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into 655c4e58c03540fbb60f72c9f647fa29(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:45,513 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:45,513 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104885428; duration=0sec 2024-11-20T12:14:45,513 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:45,513 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:45,513 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:45,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:45,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:45,521 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,521 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0518a77392b842a193da4391c9ecd41e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=36.7 K 2024-11-20T12:14:45,522 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0518a77392b842a193da4391c9ecd41e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732104880460 2024-11-20T12:14:45,523 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b05a137fdf704c7eb96e5c388224694c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1732104883017 2024-11-20T12:14:45,524 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 613cf54f194b445d835434cae9643319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883705 2024-11-20T12:14:45,550 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:45,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/86a84d8579a145559922d63f48eed53c is 50, key is test_row_0/C:col10/1732104885239/Put/seqid=0 2024-11-20T12:14:45,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T12:14:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,582 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:14:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741905_1081 (size=13051) 2024-11-20T12:14:45,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ec4e942f4b264a77a19b32cd7ac25984 is 50, key is test_row_0/A:col10/1732104885288/Put/seqid=0 2024-11-20T12:14:45,616 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/86a84d8579a145559922d63f48eed53c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/86a84d8579a145559922d63f48eed53c 2024-11-20T12:14:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:45,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:45,630 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into 86a84d8579a145559922d63f48eed53c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:45,630 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:45,630 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104885432; duration=0sec 2024-11-20T12:14:45,630 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:45,630 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:45,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741906_1082 (size=12301) 2024-11-20T12:14:45,635 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ec4e942f4b264a77a19b32cd7ac25984 2024-11-20T12:14:45,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/401488da8593474aa5bfed9e3f8c35b0 is 50, key is test_row_0/B:col10/1732104885288/Put/seqid=0 2024-11-20T12:14:45,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104945659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104945661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741907_1083 (size=12301) 2024-11-20T12:14:45,668 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/401488da8593474aa5bfed9e3f8c35b0 2024-11-20T12:14:45,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/edf5ce60bc864f8895cb48d199763a55 is 50, key is test_row_0/C:col10/1732104885288/Put/seqid=0 2024-11-20T12:14:45,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741908_1084 (size=12301) 2024-11-20T12:14:45,723 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/edf5ce60bc864f8895cb48d199763a55 2024-11-20T12:14:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T12:14:45,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ec4e942f4b264a77a19b32cd7ac25984 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984 2024-11-20T12:14:45,752 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984, entries=150, sequenceid=341, filesize=12.0 K 2024-11-20T12:14:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/401488da8593474aa5bfed9e3f8c35b0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0 2024-11-20T12:14:45,764 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0, entries=150, sequenceid=341, filesize=12.0 K 2024-11-20T12:14:45,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/edf5ce60bc864f8895cb48d199763a55 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55 2024-11-20T12:14:45,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104945767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104945767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,784 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55, entries=150, sequenceid=341, filesize=12.0 K 2024-11-20T12:14:45,788 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 205ms, sequenceid=341, compaction requested=false 2024-11-20T12:14:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T12:14:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T12:14:45,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T12:14:45,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 679 msec 2024-11-20T12:14:45,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 687 msec 2024-11-20T12:14:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:45,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:45,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:45,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/be8c7578a2a64874aa0da3cd18e7daf7 is 50, key is test_row_0/A:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:45,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741909_1085 (size=14741) 2024-11-20T12:14:45,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/be8c7578a2a64874aa0da3cd18e7daf7 2024-11-20T12:14:45,901 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/447eac8560d84b92b206d49236a58d49 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/447eac8560d84b92b206d49236a58d49 2024-11-20T12:14:45,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104945896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104945899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104945902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,922 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into 447eac8560d84b92b206d49236a58d49(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:45,922 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:45,923 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104885428; duration=0sec 2024-11-20T12:14:45,923 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:45,923 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:45,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/25fa1a6c917b4052aa0d238ed56cdb8a is 50, key is test_row_0/B:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:45,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741910_1086 (size=12301) 2024-11-20T12:14:45,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104945977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:45,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104945987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:45,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104946009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104946016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104946016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104946216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104946227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104946227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T12:14:46,247 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:46,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T12:14:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:46,251 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:46,252 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:46,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:46,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104946287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104946298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/25fa1a6c917b4052aa0d238ed56cdb8a 2024-11-20T12:14:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:46,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/e97dfd7fd1a64c14b64bd989f61cc272 is 50, key is test_row_0/C:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:46,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741911_1087 (size=12301) 2024-11-20T12:14:46,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104946527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104946536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104946536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:46,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/e97dfd7fd1a64c14b64bd989f61cc272 2024-11-20T12:14:46,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/be8c7578a2a64874aa0da3cd18e7daf7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7 2024-11-20T12:14:46,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7, entries=200, sequenceid=358, filesize=14.4 K 2024-11-20T12:14:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104946797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/25fa1a6c917b4052aa0d238ed56cdb8a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a 2024-11-20T12:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T12:14:46,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/e97dfd7fd1a64c14b64bd989f61cc272 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272 2024-11-20T12:14:46,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272, entries=150, sequenceid=358, filesize=12.0 K 2024-11-20T12:14:46,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 955ms, sequenceid=358, compaction requested=true 2024-11-20T12:14:46,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:46,816 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:46,816 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:46,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:46,817 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:46,817 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:46,817 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:46,817 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:46,817 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,817 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,817 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/447eac8560d84b92b206d49236a58d49, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=39.2 K 2024-11-20T12:14:46,817 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/655c4e58c03540fbb60f72c9f647fa29, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=36.8 K 2024-11-20T12:14:46,818 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447eac8560d84b92b206d49236a58d49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883705 2024-11-20T12:14:46,818 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 655c4e58c03540fbb60f72c9f647fa29, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883705 2024-11-20T12:14:46,819 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 401488da8593474aa5bfed9e3f8c35b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732104885280 2024-11-20T12:14:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:46,819 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec4e942f4b264a77a19b32cd7ac25984, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732104885280 2024-11-20T12:14:46,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:46,819 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 25fa1a6c917b4052aa0d238ed56cdb8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:46,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:46,820 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting be8c7578a2a64874aa0da3cd18e7daf7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:46,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c7d62a3b573f4e05b61322e81ec4ea05 is 50, key is test_row_0/A:col10/1732104885883/Put/seqid=0 2024-11-20T12:14:46,837 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:46,838 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/a9c0c9de458a445b8adb544ab6b7cf89 is 50, key is test_row_0/A:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:46,846 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:46,847 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/950e1833dd96423e9b2ea41ae59b3c7a is 50, key is test_row_0/B:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:46,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741912_1088 (size=14741) 2024-11-20T12:14:46,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c7d62a3b573f4e05b61322e81ec4ea05 2024-11-20T12:14:46,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:46,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:46,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:46,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741914_1090 (size=13153) 2024-11-20T12:14:46,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:46,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104946907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:46,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:46,917 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/950e1833dd96423e9b2ea41ae59b3c7a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/950e1833dd96423e9b2ea41ae59b3c7a 2024-11-20T12:14:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741913_1089 (size=13153) 2024-11-20T12:14:46,931 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into 950e1833dd96423e9b2ea41ae59b3c7a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:46,931 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:46,931 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=13, startTime=1732104886816; duration=0sec 2024-11-20T12:14:46,931 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:46,931 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:46,931 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:14:46,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/a2511be4ee5e4d26b7171913d3dc15be is 50, key is test_row_0/B:col10/1732104885883/Put/seqid=0 2024-11-20T12:14:46,933 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:14:46,933 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:46,933 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:46,933 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/86a84d8579a145559922d63f48eed53c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=36.8 K 2024-11-20T12:14:46,934 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a84d8579a145559922d63f48eed53c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1732104883705 2024-11-20T12:14:46,934 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting edf5ce60bc864f8895cb48d199763a55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732104885280 2024-11-20T12:14:46,934 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e97dfd7fd1a64c14b64bd989f61cc272, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:46,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741915_1091 (size=12301) 2024-11-20T12:14:46,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/a2511be4ee5e4d26b7171913d3dc15be 2024-11-20T12:14:46,982 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:46,983 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/5c818b28293f4a9dbf40a4316a4ff812 is 50, key is test_row_0/C:col10/1732104885638/Put/seqid=0 2024-11-20T12:14:46,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bf9e467478054a25a24c7f6c943002ee is 50, key is test_row_0/C:col10/1732104885883/Put/seqid=0 2024-11-20T12:14:47,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741916_1092 (size=13153) 2024-11-20T12:14:47,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104947017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,028 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/5c818b28293f4a9dbf40a4316a4ff812 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5c818b28293f4a9dbf40a4316a4ff812 2024-11-20T12:14:47,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:47,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:47,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:47,038 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into 5c818b28293f4a9dbf40a4316a4ff812(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:47,038 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:47,038 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=13, startTime=1732104886816; duration=0sec 2024-11-20T12:14:47,040 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:47,040 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:47,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741917_1093 (size=12301) 2024-11-20T12:14:47,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104947037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bf9e467478054a25a24c7f6c943002ee 2024-11-20T12:14:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104947046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104947047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c7d62a3b573f4e05b61322e81ec4ea05 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05 2024-11-20T12:14:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05, entries=200, sequenceid=376, filesize=14.4 K 2024-11-20T12:14:47,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/a2511be4ee5e4d26b7171913d3dc15be as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be 2024-11-20T12:14:47,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T12:14:47,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/bf9e467478054a25a24c7f6c943002ee as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee 2024-11-20T12:14:47,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee, entries=150, sequenceid=376, filesize=12.0 K 2024-11-20T12:14:47,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 263ms, sequenceid=376, compaction requested=false 2024-11-20T12:14:47,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:47,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T12:14:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:47,184 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:47,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:47,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/5e76a23cbca14aab90e3510b4be27bb5 is 50, key is test_row_0/A:col10/1732104886899/Put/seqid=0 2024-11-20T12:14:47,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741918_1094 (size=12301) 2024-11-20T12:14:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:47,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:47,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104947324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,334 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/a9c0c9de458a445b8adb544ab6b7cf89 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/a9c0c9de458a445b8adb544ab6b7cf89 2024-11-20T12:14:47,346 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into a9c0c9de458a445b8adb544ab6b7cf89(size=12.8 K), total size for store is 27.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:47,346 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:47,346 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=13, startTime=1732104886815; duration=0sec 2024-11-20T12:14:47,346 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:47,346 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104947436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,623 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/5e76a23cbca14aab90e3510b4be27bb5 2024-11-20T12:14:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d3f7e4ad681d4ae8b33e860c6b2b081b is 50, key is test_row_0/B:col10/1732104886899/Put/seqid=0 2024-11-20T12:14:47,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104947647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741919_1095 (size=12301) 2024-11-20T12:14:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104947806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,812 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104947957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,056 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d3f7e4ad681d4ae8b33e860c6b2b081b 2024-11-20T12:14:48,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104948056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104948056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,057 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,059 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/0d3a6b0c829e405096ce290e0aafb6bb is 50, key is test_row_0/C:col10/1732104886899/Put/seqid=0 2024-11-20T12:14:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104948067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,073 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741920_1096 (size=12301) 2024-11-20T12:14:48,091 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=393 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/0d3a6b0c829e405096ce290e0aafb6bb 2024-11-20T12:14:48,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/5e76a23cbca14aab90e3510b4be27bb5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5 2024-11-20T12:14:48,108 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T12:14:48,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d3f7e4ad681d4ae8b33e860c6b2b081b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b 2024-11-20T12:14:48,120 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T12:14:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/0d3a6b0c829e405096ce290e0aafb6bb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb 2024-11-20T12:14:48,131 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb, entries=150, sequenceid=393, filesize=12.0 K 2024-11-20T12:14:48,134 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 950ms, sequenceid=393, compaction requested=true 2024-11-20T12:14:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:48,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T12:14:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T12:14:48,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T12:14:48,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8850 sec 2024-11-20T12:14:48,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8990 sec 2024-11-20T12:14:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T12:14:48,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:48,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:14:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T12:14:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:48,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:14:48,402 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:14:48,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:14:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:48,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:48,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:48,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/7eff59b8a39a4e11b7697044cea9166b is 50, key is test_row_0/A:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:48,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741921_1097 (size=14741) 2024-11-20T12:14:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104948518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:48,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104948626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:48,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:48,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104948837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:48,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:48,842 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:48,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:48,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:48,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/7eff59b8a39a4e11b7697044cea9166b 2024-11-20T12:14:48,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ed717002d1a04295b83c3c769ffe0c01 is 50, key is test_row_0/B:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:48,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741922_1098 (size=12301) 2024-11-20T12:14:48,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ed717002d1a04295b83c3c769ffe0c01 2024-11-20T12:14:48,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/7a1f86f8f4fd4558ade347af3e59f6d4 is 50, key is test_row_0/C:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:49,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:49,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741923_1099 (size=12301) 2024-11-20T12:14:49,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/7a1f86f8f4fd4558ade347af3e59f6d4 2024-11-20T12:14:49,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/7eff59b8a39a4e11b7697044cea9166b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b 2024-11-20T12:14:49,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b, entries=200, sequenceid=412, filesize=14.4 K 2024-11-20T12:14:49,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/ed717002d1a04295b83c3c769ffe0c01 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01 2024-11-20T12:14:49,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01, entries=150, sequenceid=412, filesize=12.0 K 2024-11-20T12:14:49,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/7a1f86f8f4fd4558ade347af3e59f6d4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4 2024-11-20T12:14:49,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4, entries=150, sequenceid=412, filesize=12.0 K 2024-11-20T12:14:49,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 595ms, sequenceid=412, compaction requested=true 2024-11-20T12:14:49,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:49,065 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:49,065 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:49,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:49,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:49,066 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54936 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:49,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:49,066 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:49,066 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,066 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,066 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/a9c0c9de458a445b8adb544ab6b7cf89, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=53.6 K 2024-11-20T12:14:49,066 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/950e1833dd96423e9b2ea41ae59b3c7a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=48.9 K 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c0c9de458a445b8adb544ab6b7cf89, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 950e1833dd96423e9b2ea41ae59b3c7a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a2511be4ee5e4d26b7171913d3dc15be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732104885883 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7d62a3b573f4e05b61322e81ec4ea05, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732104885883 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e76a23cbca14aab90e3510b4be27bb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732104886893 2024-11-20T12:14:49,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d3f7e4ad681d4ae8b33e860c6b2b081b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732104886893 2024-11-20T12:14:49,068 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eff59b8a39a4e11b7697044cea9166b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:49,068 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ed717002d1a04295b83c3c769ffe0c01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:49,082 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:49,083 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/e4d07069c3a142d58f8781610054ee62 is 50, key is test_row_0/A:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:49,086 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:49,086 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d95c2aa043b646c783d9fc910c0c20cc is 50, key is test_row_0/B:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:49,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741924_1100 (size=13289) 2024-11-20T12:14:49,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741925_1101 (size=13289) 2024-11-20T12:14:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:49,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:49,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4508c9d046384dd99b5d1c1498333260 is 50, key is test_row_0/A:col10/1732104888513/Put/seqid=0 2024-11-20T12:14:49,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741926_1102 (size=14741) 2024-11-20T12:14:49,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4508c9d046384dd99b5d1c1498333260 2024-11-20T12:14:49,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4bbbed7d7dc64c278d9fc215302bff87 is 50, key is test_row_0/B:col10/1732104888513/Put/seqid=0 2024-11-20T12:14:49,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741927_1103 (size=12301) 2024-11-20T12:14:49,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104949240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:49,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104949357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:49,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,502 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/e4d07069c3a142d58f8781610054ee62 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/e4d07069c3a142d58f8781610054ee62 2024-11-20T12:14:49,506 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d95c2aa043b646c783d9fc910c0c20cc as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d95c2aa043b646c783d9fc910c0c20cc 2024-11-20T12:14:49,519 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into d95c2aa043b646c783d9fc910c0c20cc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:49,519 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:49,519 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=12, startTime=1732104889065; duration=0sec 2024-11-20T12:14:49,519 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:49,519 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:49,519 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into e4d07069c3a142d58f8781610054ee62(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:49,519 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:49,520 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:49,520 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=12, startTime=1732104889064; duration=0sec 2024-11-20T12:14:49,522 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:49,522 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:49,522 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:49,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:49,522 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5c818b28293f4a9dbf40a4316a4ff812, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=48.9 K 2024-11-20T12:14:49,523 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c818b28293f4a9dbf40a4316a4ff812, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732104885638 2024-11-20T12:14:49,523 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9e467478054a25a24c7f6c943002ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1732104885883 2024-11-20T12:14:49,523 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d3a6b0c829e405096ce290e0aafb6bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=393, earliestPutTs=1732104886893 2024-11-20T12:14:49,524 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a1f86f8f4fd4558ade347af3e59f6d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:49,552 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:49,553 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/e077de21f35a4395bf2629dd9a150b03 is 50, key is test_row_0/C:col10/1732104888468/Put/seqid=0 2024-11-20T12:14:49,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104949567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:49,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4bbbed7d7dc64c278d9fc215302bff87 2024-11-20T12:14:49,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741928_1104 (size=13289) 2024-11-20T12:14:49,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8b38aeeb2c19485cb7cc9b9d23ec701b is 50, key is test_row_0/C:col10/1732104888513/Put/seqid=0 2024-11-20T12:14:49,615 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/e077de21f35a4395bf2629dd9a150b03 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e077de21f35a4395bf2629dd9a150b03 2024-11-20T12:14:49,624 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into e077de21f35a4395bf2629dd9a150b03(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:49,624 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:49,624 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=12, startTime=1732104889065; duration=0sec 2024-11-20T12:14:49,624 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:49,624 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:49,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741929_1105 (size=12301) 2024-11-20T12:14:49,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8b38aeeb2c19485cb7cc9b9d23ec701b 2024-11-20T12:14:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4508c9d046384dd99b5d1c1498333260 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260 2024-11-20T12:14:49,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:14:49,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260, entries=200, sequenceid=427, filesize=14.4 K 2024-11-20T12:14:49,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/4bbbed7d7dc64c278d9fc215302bff87 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87 2024-11-20T12:14:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T12:14:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8b38aeeb2c19485cb7cc9b9d23ec701b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b 2024-11-20T12:14:49,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T12:14:49,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 515ms, sequenceid=427, compaction requested=false 2024-11-20T12:14:49,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:49,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:49,794 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:49,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c757433c9ddf405d90b7806fee08faf5 is 50, key is test_row_0/A:col10/1732104889195/Put/seqid=0 2024-11-20T12:14:49,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741930_1106 (size=12301) 2024-11-20T12:14:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:49,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. as already flushing 2024-11-20T12:14:49,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104949876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:49,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104949887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:49,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104949987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52314 deadline: 1732104950077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52284 deadline: 1732104950077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,082 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,083 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52304 deadline: 1732104950096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,101 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104950196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,203 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c757433c9ddf405d90b7806fee08faf5 2024-11-20T12:14:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/dd08cf8a7b5441b48cbfb280165498e0 is 50, key is test_row_0/B:col10/1732104889195/Put/seqid=0 2024-11-20T12:14:50,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741931_1107 (size=12301) 2024-11-20T12:14:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104950397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104950506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:50,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:50,629 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/dd08cf8a7b5441b48cbfb280165498e0 2024-11-20T12:14:50,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/716071a29cd94ff8a42d40fe983585e6 is 50, key is test_row_0/C:col10/1732104889195/Put/seqid=0 2024-11-20T12:14:50,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741932_1108 (size=12301) 2024-11-20T12:14:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104951026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:51,044 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/716071a29cd94ff8a42d40fe983585e6 2024-11-20T12:14:51,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/c757433c9ddf405d90b7806fee08faf5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5 2024-11-20T12:14:51,056 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T12:14:51,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/dd08cf8a7b5441b48cbfb280165498e0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0 2024-11-20T12:14:51,063 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T12:14:51,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/716071a29cd94ff8a42d40fe983585e6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6 2024-11-20T12:14:51,069 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6, entries=150, sequenceid=447, filesize=12.0 K 2024-11-20T12:14:51,070 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1276ms, sequenceid=447, compaction requested=true 2024-11-20T12:14:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T12:14:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T12:14:51,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T12:14:51,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6700 sec 2024-11-20T12:14:51,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.6780 sec 2024-11-20T12:14:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:51,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:14:51,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bfcef122e55746bcb2decd5d222f5c53 is 50, key is test_row_0/A:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741933_1109 (size=14741) 2024-11-20T12:14:51,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bfcef122e55746bcb2decd5d222f5c53 2024-11-20T12:14:51,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d9a94fb86b1a4bd4b0ff3dd4378528ae is 50, key is test_row_0/B:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741934_1110 (size=12301) 2024-11-20T12:14:51,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d9a94fb86b1a4bd4b0ff3dd4378528ae 2024-11-20T12:14:51,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/5d985cd672b24cd792bbba8d545e0d55 is 50, key is test_row_0/C:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104951499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:51,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741935_1111 (size=12301) 2024-11-20T12:14:51,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/5d985cd672b24cd792bbba8d545e0d55 2024-11-20T12:14:51,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/bfcef122e55746bcb2decd5d222f5c53 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53 2024-11-20T12:14:51,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53, entries=200, sequenceid=463, filesize=14.4 K 2024-11-20T12:14:51,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/d9a94fb86b1a4bd4b0ff3dd4378528ae as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae 2024-11-20T12:14:51,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae, entries=150, sequenceid=463, filesize=12.0 K 2024-11-20T12:14:51,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/5d985cd672b24cd792bbba8d545e0d55 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55 2024-11-20T12:14:51,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55, entries=150, sequenceid=463, filesize=12.0 K 2024-11-20T12:14:51,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 128ms, sequenceid=463, compaction requested=true 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:51,547 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:51,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:51,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:51,548 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:51,548 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:51,548 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/B is initiating minor compaction (all files) 2024-11-20T12:14:51,548 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/A is initiating minor compaction (all files) 2024-11-20T12:14:51,548 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/B in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:51,549 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d95c2aa043b646c783d9fc910c0c20cc, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=49.0 K 2024-11-20T12:14:51,549 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/A in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:51,549 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/e4d07069c3a142d58f8781610054ee62, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=53.8 K 2024-11-20T12:14:51,549 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d95c2aa043b646c783d9fc910c0c20cc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:51,552 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d07069c3a142d58f8781610054ee62, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:51,552 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bbbed7d7dc64c278d9fc215302bff87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104888504 2024-11-20T12:14:51,552 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4508c9d046384dd99b5d1c1498333260, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104888504 2024-11-20T12:14:51,552 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting dd08cf8a7b5441b48cbfb280165498e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732104889173 2024-11-20T12:14:51,552 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c757433c9ddf405d90b7806fee08faf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732104889173 2024-11-20T12:14:51,553 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a94fb86b1a4bd4b0ff3dd4378528ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732104889869 2024-11-20T12:14:51,553 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfcef122e55746bcb2decd5d222f5c53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732104889864 2024-11-20T12:14:51,571 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#A#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:51,572 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/db5aaf6abd854a94ad30631d16294cc2 is 50, key is test_row_0/A:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,601 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#B#compaction#100 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:51,602 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/da8de9907bbc49f085df93e33cd0e082 is 50, key is test_row_0/B:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:51,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:51,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741936_1112 (size=13425) 2024-11-20T12:14:51,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3d7a98fde98f49849e9e6be1cc79800f is 50, key is test_row_0/A:col10/1732104891451/Put/seqid=0 2024-11-20T12:14:51,641 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/db5aaf6abd854a94ad30631d16294cc2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/db5aaf6abd854a94ad30631d16294cc2 2024-11-20T12:14:51,652 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/A of 56b7424cfd47be723aa0ea7e4fa06ec9 into db5aaf6abd854a94ad30631d16294cc2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:51,652 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:51,652 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/A, priority=12, startTime=1732104891547; duration=0sec 2024-11-20T12:14:51,652 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:51,652 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:51,653 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:14:51,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741938_1114 (size=14741) 2024-11-20T12:14:51,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741937_1113 (size=13425) 2024-11-20T12:14:51,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:14:51,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 56b7424cfd47be723aa0ea7e4fa06ec9/C is initiating minor compaction (all files) 2024-11-20T12:14:51,661 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 56b7424cfd47be723aa0ea7e4fa06ec9/C in TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:51,661 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e077de21f35a4395bf2629dd9a150b03, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp, totalSize=49.0 K 2024-11-20T12:14:51,662 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e077de21f35a4395bf2629dd9a150b03, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732104887278 2024-11-20T12:14:51,663 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b38aeeb2c19485cb7cc9b9d23ec701b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104888504 2024-11-20T12:14:51,663 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 716071a29cd94ff8a42d40fe983585e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1732104889173 2024-11-20T12:14:51,664 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d985cd672b24cd792bbba8d545e0d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732104889869 2024-11-20T12:14:51,678 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 56b7424cfd47be723aa0ea7e4fa06ec9#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:14:51,679 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/247b6e98bb5f43a0aedaeb08d809bdb1 is 50, key is test_row_0/C:col10/1732104889874/Put/seqid=0 2024-11-20T12:14:51,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741939_1115 (size=13425) 2024-11-20T12:14:51,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104951699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104951808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:52,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:52,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104952016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:52,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:52,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52292 deadline: 1732104952047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:52,049 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 56b7424cfd47be723aa0ea7e4fa06ec9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:14:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:52,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3d7a98fde98f49849e9e6be1cc79800f 2024-11-20T12:14:52,068 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/da8de9907bbc49f085df93e33cd0e082 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/da8de9907bbc49f085df93e33cd0e082 2024-11-20T12:14:52,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/3a4dd8bad5b84d3daa81b6e16e4460da is 50, key is test_row_0/B:col10/1732104891451/Put/seqid=0 2024-11-20T12:14:52,074 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/B of 56b7424cfd47be723aa0ea7e4fa06ec9 into da8de9907bbc49f085df93e33cd0e082(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:52,074 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:52,074 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/B, priority=12, startTime=1732104891547; duration=0sec 2024-11-20T12:14:52,074 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:52,074 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:52,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741940_1116 (size=12301) 2024-11-20T12:14:52,094 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/247b6e98bb5f43a0aedaeb08d809bdb1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/247b6e98bb5f43a0aedaeb08d809bdb1 2024-11-20T12:14:52,101 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 56b7424cfd47be723aa0ea7e4fa06ec9/C of 56b7424cfd47be723aa0ea7e4fa06ec9 into 247b6e98bb5f43a0aedaeb08d809bdb1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:14:52,101 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:52,102 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., storeName=56b7424cfd47be723aa0ea7e4fa06ec9/C, priority=12, startTime=1732104891547; duration=0sec 2024-11-20T12:14:52,102 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:52,102 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:52,208 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T12:14:52,208 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:52,211 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T12:14:52,211 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:52,212 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T12:14:52,212 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:52,214 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,214 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,214 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,214 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,214 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:52,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:52,214 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:52,214 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,219 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T12:14:52,219 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:52,219 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,219 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:52,219 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:52,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104952326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:52,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/3a4dd8bad5b84d3daa81b6e16e4460da 2024-11-20T12:14:52,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/11af671a22494cdba9104a6f6373fc42 is 50, key is test_row_0/C:col10/1732104891451/Put/seqid=0 2024-11-20T12:14:52,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741941_1117 (size=12301) 2024-11-20T12:14:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T12:14:52,557 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:52,704 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:14:52,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:14:52,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:52266 deadline: 1732104952836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:14:52,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:14:52,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/11af671a22494cdba9104a6f6373fc42 2024-11-20T12:14:52,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/3d7a98fde98f49849e9e6be1cc79800f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3d7a98fde98f49849e9e6be1cc79800f 2024-11-20T12:14:52,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3d7a98fde98f49849e9e6be1cc79800f, entries=200, sequenceid=480, filesize=14.4 K 2024-11-20T12:14:52,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/3a4dd8bad5b84d3daa81b6e16e4460da as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/3a4dd8bad5b84d3daa81b6e16e4460da 2024-11-20T12:14:52,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/3a4dd8bad5b84d3daa81b6e16e4460da, entries=150, sequenceid=480, filesize=12.0 K 2024-11-20T12:14:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/11af671a22494cdba9104a6f6373fc42 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/11af671a22494cdba9104a6f6373fc42 2024-11-20T12:14:52,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/11af671a22494cdba9104a6f6373fc42, entries=150, sequenceid=480, filesize=12.0 K 2024-11-20T12:14:52,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1316ms, sequenceid=480, compaction requested=false 2024-11-20T12:14:52,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:53,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:53,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:14:53,848 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:53,848 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:53,848 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:53,848 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:53,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:53,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:53,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ac6317018e994bd6bd40454cdfb65c63 is 50, key is test_row_0/A:col10/1732104891695/Put/seqid=0 2024-11-20T12:14:53,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741942_1118 (size=12301) 2024-11-20T12:14:54,059 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T12:14:54,059 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:54,059 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,059 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:54,152 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T12:14:54,152 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T12:14:54,152 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T12:14:54,152 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:54,152 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:54,152 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:14:54,152 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:54,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:54,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:54,152 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2514 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2535 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1126 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3361 rows 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1123 2024-11-20T12:14:54,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3356 rows 2024-11-20T12:14:54,153 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:14:54,153 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:14:54,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:14:54,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:14:54,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:14:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:54,163 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104894163"}]},"ts":"1732104894163"} 2024-11-20T12:14:54,165 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:14:54,165 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:14:54,167 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:14:54,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, UNASSIGN}] 2024-11-20T12:14:54,172 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, UNASSIGN 2024-11-20T12:14:54,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=56b7424cfd47be723aa0ea7e4fa06ec9, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:54,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, UNASSIGN because future has completed 2024-11-20T12:14:54,175 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:14:54,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:14:54,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ac6317018e994bd6bd40454cdfb65c63 2024-11-20T12:14:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:54,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f4daaa42d8cf4c04ba3cf01fd7f22310 is 50, key is test_row_0/B:col10/1732104891695/Put/seqid=0 2024-11-20T12:14:54,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741943_1119 (size=12301) 2024-11-20T12:14:54,332 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:54,332 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:14:54,332 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 56b7424cfd47be723aa0ea7e4fa06ec9, disabling compactions & flushes 2024-11-20T12:14:54,332 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:54,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f4daaa42d8cf4c04ba3cf01fd7f22310 2024-11-20T12:14:54,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8a85ca2191374248a6a0b0603d7c1b69 is 50, key is test_row_0/C:col10/1732104891695/Put/seqid=0 2024-11-20T12:14:54,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741944_1120 (size=12301) 2024-11-20T12:14:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:55,102 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8a85ca2191374248a6a0b0603d7c1b69 2024-11-20T12:14:55,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/ac6317018e994bd6bd40454cdfb65c63 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ac6317018e994bd6bd40454cdfb65c63 2024-11-20T12:14:55,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ac6317018e994bd6bd40454cdfb65c63, entries=150, sequenceid=499, filesize=12.0 K 2024-11-20T12:14:55,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/f4daaa42d8cf4c04ba3cf01fd7f22310 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4daaa42d8cf4c04ba3cf01fd7f22310 2024-11-20T12:14:55,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4daaa42d8cf4c04ba3cf01fd7f22310, entries=150, sequenceid=499, filesize=12.0 K 2024-11-20T12:14:55,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/8a85ca2191374248a6a0b0603d7c1b69 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8a85ca2191374248a6a0b0603d7c1b69 2024-11-20T12:14:55,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8a85ca2191374248a6a0b0603d7c1b69, entries=150, sequenceid=499, filesize=12.0 K 2024-11-20T12:14:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1281ms, sequenceid=499, compaction requested=true 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 56b7424cfd47be723aa0ea7e4fa06ec9: 2024-11-20T12:14:55,129 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:14:55,129 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:55,129 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. after waiting 0 ms 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:14:55,129 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:14:55,129 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. because compaction request was cancelled 2024-11-20T12:14:55,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 56b7424cfd47be723aa0ea7e4fa06ec9:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:14:55,129 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:A 2024-11-20T12:14:55,130 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. because compaction request was cancelled 2024-11-20T12:14:55,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:14:55,130 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 56b7424cfd47be723aa0ea7e4fa06ec9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T12:14:55,130 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. because compaction request was cancelled 2024-11-20T12:14:55,130 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:B 2024-11-20T12:14:55,130 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 56b7424cfd47be723aa0ea7e4fa06ec9:C 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=A 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=B 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 56b7424cfd47be723aa0ea7e4fa06ec9, store=C 2024-11-20T12:14:55,130 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:14:55,135 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4f4f087a60e94aadb2a96e2596da1828 is 50, key is test_row_0/A:col10/1732104894117/Put/seqid=0 2024-11-20T12:14:55,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741945_1121 (size=9857) 2024-11-20T12:14:55,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:55,540 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4f4f087a60e94aadb2a96e2596da1828 2024-11-20T12:14:55,547 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/42ede5e74a85496bb285371e931e3eae is 50, key is test_row_0/B:col10/1732104894117/Put/seqid=0 2024-11-20T12:14:55,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741946_1122 (size=9857) 2024-11-20T12:14:55,953 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/42ede5e74a85496bb285371e931e3eae 2024-11-20T12:14:55,970 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/141ce20d2760439f8f64481e670e780c is 50, key is test_row_0/C:col10/1732104894117/Put/seqid=0 2024-11-20T12:14:55,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741947_1123 (size=9857) 2024-11-20T12:14:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:56,375 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=506 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/141ce20d2760439f8f64481e670e780c 2024-11-20T12:14:56,380 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/A/4f4f087a60e94aadb2a96e2596da1828 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4f4f087a60e94aadb2a96e2596da1828 2024-11-20T12:14:56,384 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4f4f087a60e94aadb2a96e2596da1828, entries=100, sequenceid=506, filesize=9.6 K 2024-11-20T12:14:56,385 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/B/42ede5e74a85496bb285371e931e3eae as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/42ede5e74a85496bb285371e931e3eae 2024-11-20T12:14:56,389 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/42ede5e74a85496bb285371e931e3eae, entries=100, sequenceid=506, filesize=9.6 K 2024-11-20T12:14:56,390 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/.tmp/C/141ce20d2760439f8f64481e670e780c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/141ce20d2760439f8f64481e670e780c 2024-11-20T12:14:56,395 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/141ce20d2760439f8f64481e670e780c, entries=100, sequenceid=506, filesize=9.6 K 2024-11-20T12:14:56,395 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1266ms, sequenceid=506, compaction requested=true 2024-11-20T12:14:56,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ea909f4d6d6e429c9e10edacf11f7611, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/1ec865a2b1ba47349730108b999f0fb9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3f99ff146dfd48f39fcd7f381229a232, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b6e82b4d5f3e4664801fdfc69483ada7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/21fa049611834f9596949aee27770dc6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/8715987d308644edbca242eef38da853, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/447eac8560d84b92b206d49236a58d49, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/a9c0c9de458a445b8adb544ab6b7cf89, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/e4d07069c3a142d58f8781610054ee62, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53] to archive 2024-11-20T12:14:56,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:14:56,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0a848ecbb4274652a22286e2bcb86ea0 2024-11-20T12:14:56,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/637a0aeaac4547b791673cb6a2413a7d 2024-11-20T12:14:56,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b12e3dcf52914663968f229bb14fba8e 2024-11-20T12:14:56,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ea909f4d6d6e429c9e10edacf11f7611 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ea909f4d6d6e429c9e10edacf11f7611 2024-11-20T12:14:56,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/9007224ab4b54e7db1510fd25c82b3e0 2024-11-20T12:14:56,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/1ec865a2b1ba47349730108b999f0fb9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/1ec865a2b1ba47349730108b999f0fb9 2024-11-20T12:14:56,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/729bea8e70c74e64b7c7d38bc0a4deb7 2024-11-20T12:14:56,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/6c875641dbe64d9eac7822925de5b7c7 2024-11-20T12:14:56,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b0c8e90824ec45abbc1fa99088666bde 2024-11-20T12:14:56,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3f99ff146dfd48f39fcd7f381229a232 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3f99ff146dfd48f39fcd7f381229a232 2024-11-20T12:14:56,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c006c93ea3b1407289a8f19aaebfc7e2 2024-11-20T12:14:56,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/cf5d4d7e2dac4653a9c93b16ecec5a5d 2024-11-20T12:14:56,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b6e82b4d5f3e4664801fdfc69483ada7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b6e82b4d5f3e4664801fdfc69483ada7 2024-11-20T12:14:56,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2e74fe761bd74e85a3ee6f28c66e508b 2024-11-20T12:14:56,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/b70826f63b814f6fab4b49fafbb62116 2024-11-20T12:14:56,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/21fa049611834f9596949aee27770dc6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/21fa049611834f9596949aee27770dc6 2024-11-20T12:14:56,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/0e4c86694a31450abb14080b3fa61d1c 2024-11-20T12:14:56,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/2d5d0c20458847218e517b85cc18b94f 2024-11-20T12:14:56,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3e1a204b1f6640ef85716af0a3173100 2024-11-20T12:14:56,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/8715987d308644edbca242eef38da853 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/8715987d308644edbca242eef38da853 2024-11-20T12:14:56,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/eec085caab654e9f8e09122206f3ae0f 2024-11-20T12:14:56,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bf5004d00055435ab41eafe0e12b50cd 2024-11-20T12:14:56,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/447eac8560d84b92b206d49236a58d49 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/447eac8560d84b92b206d49236a58d49 2024-11-20T12:14:56,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ec4e942f4b264a77a19b32cd7ac25984 2024-11-20T12:14:56,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/be8c7578a2a64874aa0da3cd18e7daf7 2024-11-20T12:14:56,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/a9c0c9de458a445b8adb544ab6b7cf89 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/a9c0c9de458a445b8adb544ab6b7cf89 2024-11-20T12:14:56,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c7d62a3b573f4e05b61322e81ec4ea05 2024-11-20T12:14:56,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/5e76a23cbca14aab90e3510b4be27bb5 2024-11-20T12:14:56,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/7eff59b8a39a4e11b7697044cea9166b 2024-11-20T12:14:56,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/e4d07069c3a142d58f8781610054ee62 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/e4d07069c3a142d58f8781610054ee62 2024-11-20T12:14:56,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4508c9d046384dd99b5d1c1498333260 2024-11-20T12:14:56,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/c757433c9ddf405d90b7806fee08faf5 2024-11-20T12:14:56,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/bfcef122e55746bcb2decd5d222f5c53 2024-11-20T12:14:56,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f8243af4032448deb20269eb9538b747, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5f0ee78a43cd45e8818a286dbecca72c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4a712dde22f4d3984e0fb5e1cd7d807, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b717d7de182a42c98bb4716600f4f590, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e9970552882c4ea4a1b673c9ccaa31e9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f69e47f6011a4c3eb0fde27aad027141, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/655c4e58c03540fbb60f72c9f647fa29, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/950e1833dd96423e9b2ea41ae59b3c7a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d95c2aa043b646c783d9fc910c0c20cc, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae] to archive 2024-11-20T12:14:56,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:14:56,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d0676124246245ddacd30a24e7d2bc8c 2024-11-20T12:14:56,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5d221cde3b514b0aa51c7ba5a00a7880 2024-11-20T12:14:56,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f8243af4032448deb20269eb9538b747 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f8243af4032448deb20269eb9538b747 2024-11-20T12:14:56,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/76ef1f9df42a4f2e9b018ceeddab0e05 2024-11-20T12:14:56,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/abb48da73feb4b7ba23c32c0992e0cbe 2024-11-20T12:14:56,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5f0ee78a43cd45e8818a286dbecca72c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/5f0ee78a43cd45e8818a286dbecca72c 2024-11-20T12:14:56,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e34f8529814f5a97d054736230541e 2024-11-20T12:14:56,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/54af9999452540eea77c3c11f4900f3b 2024-11-20T12:14:56,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4a712dde22f4d3984e0fb5e1cd7d807 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4a712dde22f4d3984e0fb5e1cd7d807 2024-11-20T12:14:56,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/7d4476ae63ec475a96563b23e8ecdb69 2024-11-20T12:14:56,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b1264092ee0f4b75b85f12ef4cf74db7 2024-11-20T12:14:56,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b717d7de182a42c98bb4716600f4f590 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/b717d7de182a42c98bb4716600f4f590 2024-11-20T12:14:56,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e3e7aec9f728411da8308d0f7bffc399 2024-11-20T12:14:56,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4a6ada9808ab44989585e56cf5142673 2024-11-20T12:14:56,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/03f481d6270d4d6fa121753e4f560de4 2024-11-20T12:14:56,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e9970552882c4ea4a1b673c9ccaa31e9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/e9970552882c4ea4a1b673c9ccaa31e9 2024-11-20T12:14:56,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f57c9b47034e4322a0127c6c33ba794d 2024-11-20T12:14:56,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d82ad9f81ea54873a915a23dfe632923 2024-11-20T12:14:56,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f69e47f6011a4c3eb0fde27aad027141 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f69e47f6011a4c3eb0fde27aad027141 2024-11-20T12:14:56,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/2b587fb702474087bb650a785f7e098f 2024-11-20T12:14:56,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/c1a92321c14b411e8ba36150f395757f 2024-11-20T12:14:56,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/655c4e58c03540fbb60f72c9f647fa29 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/655c4e58c03540fbb60f72c9f647fa29 2024-11-20T12:14:56,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ce50c53ad27a4a5fba0f434d75d441d9 2024-11-20T12:14:56,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/401488da8593474aa5bfed9e3f8c35b0 2024-11-20T12:14:56,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/950e1833dd96423e9b2ea41ae59b3c7a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/950e1833dd96423e9b2ea41ae59b3c7a 2024-11-20T12:14:56,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/25fa1a6c917b4052aa0d238ed56cdb8a 2024-11-20T12:14:56,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/a2511be4ee5e4d26b7171913d3dc15be 2024-11-20T12:14:56,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d3f7e4ad681d4ae8b33e860c6b2b081b 2024-11-20T12:14:56,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d95c2aa043b646c783d9fc910c0c20cc to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d95c2aa043b646c783d9fc910c0c20cc 2024-11-20T12:14:56,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/ed717002d1a04295b83c3c769ffe0c01 2024-11-20T12:14:56,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/4bbbed7d7dc64c278d9fc215302bff87 2024-11-20T12:14:56,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/dd08cf8a7b5441b48cbfb280165498e0 2024-11-20T12:14:56,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/d9a94fb86b1a4bd4b0ff3dd4378528ae 2024-11-20T12:14:56,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fb6ec65e26494540a37b5985235b45e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c12f581d6061481da3232e3304802bbf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/16539d5976554be9b3a37cd90d640897, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/df4cdb3aeb424566baf0c611e766e666, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c4035dfadb2248af8f5c01c922cc3a0f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0518a77392b842a193da4391c9ecd41e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/86a84d8579a145559922d63f48eed53c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5c818b28293f4a9dbf40a4316a4ff812, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e077de21f35a4395bf2629dd9a150b03, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55] to archive 2024-11-20T12:14:56,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:14:56,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f52ee3fc891d484c8237b3abce0ba4c8 2024-11-20T12:14:56,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/abae925f659c44f38e7b6998bb6515e1 2024-11-20T12:14:56,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fb6ec65e26494540a37b5985235b45e0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fb6ec65e26494540a37b5985235b45e0 2024-11-20T12:14:56,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b63b775636cf470e9e4fdda5f551bce4 2024-11-20T12:14:56,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/4c6b33a4756142dd98b7ef4914ec81de 2024-11-20T12:14:56,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c12f581d6061481da3232e3304802bbf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c12f581d6061481da3232e3304802bbf 2024-11-20T12:14:56,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/6a3fad21d7224bcfad5871d5bc1b3397 2024-11-20T12:14:56,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/beb01876d27a4c7d935f1c1023c71fc6 2024-11-20T12:14:56,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/16539d5976554be9b3a37cd90d640897 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/16539d5976554be9b3a37cd90d640897 2024-11-20T12:14:56,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/fe1fb3abbd324801b456e0d0f25c1eed 2024-11-20T12:14:56,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/cc8405197f1d43b88daa8d3fbb471164 2024-11-20T12:14:56,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/df4cdb3aeb424566baf0c611e766e666 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/df4cdb3aeb424566baf0c611e766e666 2024-11-20T12:14:56,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/ae8b564adb9944acb0d55f27184c4040 2024-11-20T12:14:56,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/2123803a8a9f4f5fb39f53b5dfb279cf 2024-11-20T12:14:56,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bd00e533040c465cba506e78dde585f4 2024-11-20T12:14:56,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c4035dfadb2248af8f5c01c922cc3a0f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c4035dfadb2248af8f5c01c922cc3a0f 2024-11-20T12:14:56,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/f3cb1b83d7b14f5aa0f468adc8b377ed 2024-11-20T12:14:56,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/00559e1690614d0ab1654d0a5115b940 2024-11-20T12:14:56,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0518a77392b842a193da4391c9ecd41e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0518a77392b842a193da4391c9ecd41e 2024-11-20T12:14:56,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/c514179ae94f429196c34b305650d045 2024-11-20T12:14:56,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/b05a137fdf704c7eb96e5c388224694c 2024-11-20T12:14:56,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/86a84d8579a145559922d63f48eed53c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/86a84d8579a145559922d63f48eed53c 2024-11-20T12:14:56,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/613cf54f194b445d835434cae9643319 2024-11-20T12:14:56,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/edf5ce60bc864f8895cb48d199763a55 2024-11-20T12:14:56,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5c818b28293f4a9dbf40a4316a4ff812 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5c818b28293f4a9dbf40a4316a4ff812 2024-11-20T12:14:56,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e97dfd7fd1a64c14b64bd989f61cc272 2024-11-20T12:14:56,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/bf9e467478054a25a24c7f6c943002ee 2024-11-20T12:14:56,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/0d3a6b0c829e405096ce290e0aafb6bb 2024-11-20T12:14:56,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e077de21f35a4395bf2629dd9a150b03 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/e077de21f35a4395bf2629dd9a150b03 2024-11-20T12:14:56,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/7a1f86f8f4fd4558ade347af3e59f6d4 2024-11-20T12:14:56,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8b38aeeb2c19485cb7cc9b9d23ec701b 2024-11-20T12:14:56,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/716071a29cd94ff8a42d40fe983585e6 2024-11-20T12:14:56,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/5d985cd672b24cd792bbba8d545e0d55 2024-11-20T12:14:56,581 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/recovered.edits/509.seqid, newMaxSeqId=509, maxSeqId=1 2024-11-20T12:14:56,581 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. 2024-11-20T12:14:56,582 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 56b7424cfd47be723aa0ea7e4fa06ec9: Waiting for close lock at 1732104894332Running coprocessor pre-close hooks at 1732104894332Disabling compacts and flushes for region at 1732104894332Disabling writes for close at 1732104895129 (+797 ms)Obtaining lock to block concurrent updates at 1732104895130 (+1 ms)Preparing flush snapshotting stores in 56b7424cfd47be723aa0ea7e4fa06ec9 at 1732104895130Finished memstore snapshotting TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732104895130Flushing stores of TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9. at 1732104895131 (+1 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/A: creating writer at 1732104895131Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/A: appending metadata at 1732104895134 (+3 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/A: closing flushed file at 1732104895134Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/B: creating writer at 1732104895544 (+410 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/B: appending metadata at 1732104895547 (+3 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/B: closing flushed file at 1732104895547Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/C: creating writer at 1732104895965 (+418 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/C: appending metadata at 1732104895969 (+4 ms)Flushing 56b7424cfd47be723aa0ea7e4fa06ec9/C: closing flushed file at 1732104895969Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e7c19e: reopening flushed file at 1732104896379 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@431cea0b: reopening flushed file at 1732104896384 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4c52c0: reopening flushed file at 1732104896389 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 56b7424cfd47be723aa0ea7e4fa06ec9 in 1266ms, sequenceid=506, compaction requested=true at 1732104896395 (+6 ms)Writing region close event to WAL at 1732104896576 (+181 ms)Running coprocessor post-close hooks at 1732104896581 (+5 ms)Closed at 1732104896581 2024-11-20T12:14:56,585 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:56,585 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=56b7424cfd47be723aa0ea7e4fa06ec9, regionState=CLOSED 2024-11-20T12:14:56,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:14:56,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T12:14:56,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 56b7424cfd47be723aa0ea7e4fa06ec9, server=843d83d51ba0,42825,1732104864746 in 2.4150 sec 2024-11-20T12:14:56,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-20T12:14:56,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=56b7424cfd47be723aa0ea7e4fa06ec9, UNASSIGN in 2.4210 sec 2024-11-20T12:14:56,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T12:14:56,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4300 sec 2024-11-20T12:14:56,601 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104896601"}]},"ts":"1732104896601"} 2024-11-20T12:14:56,603 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:14:56,603 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:14:56,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4460 sec 2024-11-20T12:14:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T12:14:58,317 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:58,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:14:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,329 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T12:14:58,331 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,336 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:58,340 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/recovered.edits] 2024-11-20T12:14:58,343 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3d7a98fde98f49849e9e6be1cc79800f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/3d7a98fde98f49849e9e6be1cc79800f 2024-11-20T12:14:58,345 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4f4f087a60e94aadb2a96e2596da1828 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/4f4f087a60e94aadb2a96e2596da1828 2024-11-20T12:14:58,346 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ac6317018e994bd6bd40454cdfb65c63 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/ac6317018e994bd6bd40454cdfb65c63 2024-11-20T12:14:58,347 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/db5aaf6abd854a94ad30631d16294cc2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/A/db5aaf6abd854a94ad30631d16294cc2 2024-11-20T12:14:58,349 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/3a4dd8bad5b84d3daa81b6e16e4460da to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/3a4dd8bad5b84d3daa81b6e16e4460da 2024-11-20T12:14:58,351 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/42ede5e74a85496bb285371e931e3eae to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/42ede5e74a85496bb285371e931e3eae 2024-11-20T12:14:58,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/da8de9907bbc49f085df93e33cd0e082 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/da8de9907bbc49f085df93e33cd0e082 2024-11-20T12:14:58,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4daaa42d8cf4c04ba3cf01fd7f22310 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/B/f4daaa42d8cf4c04ba3cf01fd7f22310 2024-11-20T12:14:58,356 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/11af671a22494cdba9104a6f6373fc42 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/11af671a22494cdba9104a6f6373fc42 2024-11-20T12:14:58,357 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/141ce20d2760439f8f64481e670e780c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/141ce20d2760439f8f64481e670e780c 2024-11-20T12:14:58,358 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/247b6e98bb5f43a0aedaeb08d809bdb1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/247b6e98bb5f43a0aedaeb08d809bdb1 2024-11-20T12:14:58,359 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8a85ca2191374248a6a0b0603d7c1b69 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/C/8a85ca2191374248a6a0b0603d7c1b69 2024-11-20T12:14:58,362 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/recovered.edits/509.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9/recovered.edits/509.seqid 2024-11-20T12:14:58,363 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/56b7424cfd47be723aa0ea7e4fa06ec9 2024-11-20T12:14:58,363 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:14:58,368 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T12:14:58,378 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:14:58,415 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:14:58,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,417 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:14:58,417 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732104898417"}]},"ts":"9223372036854775807"} 2024-11-20T12:14:58,421 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:14:58,421 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 56b7424cfd47be723aa0ea7e4fa06ec9, NAME => 'TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:14:58,421 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:14:58,422 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732104898421"}]},"ts":"9223372036854775807"} 2024-11-20T12:14:58,424 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:14:58,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-20T12:14:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-20T12:14:58,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:14:58,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:14:58,449 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1709011412_22 at /127.0.0.1:38196 [Waiting for operation #335] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2022882310_22 at /127.0.0.1:47724 [Waiting for operation #357] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS:0;843d83d51ba0:42825-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 432) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2750 (was 1703) - AvailableMemoryMB LEAK? - 2024-11-20T12:14:58,460 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=2750 2024-11-20T12:14:58,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:14:58,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:14:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:14:58,466 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:14:58,466 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:58,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-20T12:14:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:14:58,467 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:14:58,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741948_1124 (size=1061) 2024-11-20T12:14:58,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:14:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:14:58,877 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:14:58,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741949_1125 (size=53) 2024-11-20T12:14:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:14:59,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:59,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 009722d6d4375a43f57af7eff44b236c, disabling compactions & flushes 2024-11-20T12:14:59,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. after waiting 0 ms 2024-11-20T12:14:59,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,291 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 009722d6d4375a43f57af7eff44b236c: Waiting for close lock at 1732104899290Disabling compacts and flushes for region at 1732104899290Disabling writes for close at 1732104899290Writing region close event to WAL at 1732104899290Closed at 1732104899290 2024-11-20T12:14:59,294 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:14:59,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732104899294"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104899294"}]},"ts":"1732104899294"} 2024-11-20T12:14:59,297 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:14:59,298 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:14:59,299 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104899299"}]},"ts":"1732104899299"} 2024-11-20T12:14:59,300 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:14:59,301 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, ASSIGN}] 2024-11-20T12:14:59,303 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, ASSIGN 2024-11-20T12:14:59,304 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:14:59,455 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:59,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, ASSIGN because future has completed 2024-11-20T12:14:59,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:14:59,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:14:59,620 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,620 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:14:59,621 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,621 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:14:59,621 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,621 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,623 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,625 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:59,626 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName A 2024-11-20T12:14:59,627 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:59,628 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:59,628 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,630 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:59,630 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName B 2024-11-20T12:14:59,630 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:59,631 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:59,631 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,632 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:14:59,632 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName C 2024-11-20T12:14:59,632 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:14:59,633 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:14:59,633 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,633 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,634 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,634 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,636 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,636 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,636 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:14:59,637 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,640 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:14:59,640 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 009722d6d4375a43f57af7eff44b236c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71830139, jitterRate=0.0703524798154831}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:14:59,640 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:14:59,641 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 009722d6d4375a43f57af7eff44b236c: Running coprocessor pre-open hook at 1732104899621Writing region info on filesystem at 1732104899621Initializing all the Stores at 1732104899623 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104899623Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104899623Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104899623Cleaning up temporary data from old regions at 1732104899636 (+13 ms)Running coprocessor post-open hooks at 1732104899640 (+4 ms)Region opened successfully at 1732104899641 (+1 ms) 2024-11-20T12:14:59,642 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., pid=32, masterSystemTime=1732104899612 2024-11-20T12:14:59,644 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,644 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:14:59,645 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:14:59,647 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:14:59,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-20T12:14:59,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 in 188 msec 2024-11-20T12:14:59,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T12:14:59,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, ASSIGN in 349 msec 2024-11-20T12:14:59,652 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:14:59,653 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104899652"}]},"ts":"1732104899652"} 2024-11-20T12:14:59,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:14:59,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:14:59,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1920 sec 2024-11-20T12:15:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-20T12:15:00,618 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:00,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@705aba3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:00,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:00,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:00,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:00,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295e4063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:00,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:00,628 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b24555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:00,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:00,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:00,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:00,634 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bf6b1e5 2024-11-20T12:15:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:15:00,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:15:00,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:15:00,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:15:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741950_1126 (size=1097) 2024-11-20T12:15:01,065 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T12:15:01,066 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T12:15:01,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:15:01,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE}] 2024-11-20T12:15:01,082 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE 2024-11-20T12:15:01,083 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE because future has completed 2024-11-20T12:15:01,086 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:15:01,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:01,240 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,240 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:15:01,241 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 009722d6d4375a43f57af7eff44b236c, disabling compactions & flushes 2024-11-20T12:15:01,241 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,241 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,241 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. after waiting 0 ms 2024-11-20T12:15:01,241 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,249 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:15:01,250 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,250 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 009722d6d4375a43f57af7eff44b236c: Waiting for close lock at 1732104901241Running coprocessor pre-close hooks at 1732104901241Disabling compacts and flushes for region at 1732104901241Disabling writes for close at 1732104901241Writing region close event to WAL at 1732104901242 (+1 ms)Running coprocessor post-close hooks at 1732104901250 (+8 ms)Closed at 1732104901250 2024-11-20T12:15:01,250 WARN [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 009722d6d4375a43f57af7eff44b236c to self. 2024-11-20T12:15:01,253 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,253 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=CLOSED 2024-11-20T12:15:01,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:01,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T12:15:01,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 in 170 msec 2024-11-20T12:15:01,259 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE; state=CLOSED, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=true 2024-11-20T12:15:01,410 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE because future has completed 2024-11-20T12:15:01,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:01,573 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,573 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:15:01,574 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,575 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:01,575 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,575 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,578 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,579 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:01,584 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName A 2024-11-20T12:15:01,587 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:01,587 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:01,588 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,588 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:01,588 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName B 2024-11-20T12:15:01,589 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:01,589 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:01,589 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,590 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:01,590 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 009722d6d4375a43f57af7eff44b236c columnFamilyName C 2024-11-20T12:15:01,590 DEBUG [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:01,590 INFO [StoreOpener-009722d6d4375a43f57af7eff44b236c-1 {}] regionserver.HStore(327): Store=009722d6d4375a43f57af7eff44b236c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:01,591 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,591 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,592 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,593 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,594 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,594 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,594 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:15:01,596 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,597 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 009722d6d4375a43f57af7eff44b236c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73279312, jitterRate=0.09194684028625488}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:15:01,597 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,598 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 009722d6d4375a43f57af7eff44b236c: Running coprocessor pre-open hook at 1732104901575Writing region info on filesystem at 1732104901575Initializing all the Stores at 1732104901576 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104901576Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104901577 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104901577Cleaning up temporary data from old regions at 1732104901594 (+17 ms)Running coprocessor post-open hooks at 1732104901597 (+3 ms)Region opened successfully at 1732104901598 (+1 ms) 2024-11-20T12:15:01,599 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., pid=37, masterSystemTime=1732104901565 2024-11-20T12:15:01,601 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,601 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,602 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=OPEN, openSeqNum=5, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:01,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-20T12:15:01,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 in 194 msec 2024-11-20T12:15:01,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-20T12:15:01,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, REOPEN/MOVE in 527 msec 2024-11-20T12:15:01,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-20T12:15:01,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 538 msec 2024-11-20T12:15:01,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 971 msec 2024-11-20T12:15:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-20T12:15:01,619 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:01,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13460079, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,622 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a297d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,624 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0f283e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,626 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bddf7fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21bc0752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70008798, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@116f28ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c7c85e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e98764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71217950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0ab7ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,636 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a27586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3dd586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1987d108, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,640 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f96a5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b57279f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5fd39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f42cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e409a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18690ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0ba3a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,649 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e0e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e58620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:01,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:01,651 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:01,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dffd523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:01,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:01,654 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:01,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:01,656 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,656 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,656 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:01,656 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,657 DEBUG [Thread-644 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,657 DEBUG [Thread-644 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,664 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,667 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-20T12:15:01,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,670 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T12:15:01,670 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,671 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:01,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:01,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,672 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,673 DEBUG [Thread-648 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,674 DEBUG [Thread-648 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:01,693 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,704 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,707 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,710 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:01,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:01,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104901705, max timestamp=1732104901728Num uniques 859; 2024-11-20T12:15:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104901705, max timestamp=1732104901728Num uniques 859; 2024-11-20T12:15:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:01,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104901705, max timestamp=1732104901728Num uniques 859; 2024-11-20T12:15:01,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:01,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b31800693fc04c9d95417aed09db0bad_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104901674/Put/seqid=0 2024-11-20T12:15:01,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741951_1127 (size=14594) 2024-11-20T12:15:01,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:01,776 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b31800693fc04c9d95417aed09db0bad_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31800693fc04c9d95417aed09db0bad_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T12:15:01,777 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/06a3bd69263b4376b61a5ec6a01668c3, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:01,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104961764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104961769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104961771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104961776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104961780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/06a3bd69263b4376b61a5ec6a01668c3 is 175, key is test_row_0/A:col10/1732104901674/Put/seqid=0 2024-11-20T12:15:01,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741952_1128 (size=39545) 2024-11-20T12:15:01,799 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/06a3bd69263b4376b61a5ec6a01668c3 2024-11-20T12:15:01,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T12:15:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:01,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b336c19735c74f309df8e4a286b2ddd7 is 50, key is test_row_0/B:col10/1732104901674/Put/seqid=0 2024-11-20T12:15:01,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741953_1129 (size=9657) 2024-11-20T12:15:01,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b336c19735c74f309df8e4a286b2ddd7 2024-11-20T12:15:01,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104961886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104961887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104961887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104961887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104961891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:01,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:01,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/21e1b508ae5f437f87875860b7022c4a is 50, key is test_row_0/C:col10/1732104901674/Put/seqid=0 2024-11-20T12:15:01,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741954_1130 (size=9657) 2024-11-20T12:15:01,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/21e1b508ae5f437f87875860b7022c4a 2024-11-20T12:15:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/06a3bd69263b4376b61a5ec6a01668c3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3 2024-11-20T12:15:01,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3, entries=200, sequenceid=14, filesize=38.6 K 2024-11-20T12:15:01,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b336c19735c74f309df8e4a286b2ddd7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7 2024-11-20T12:15:01,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7, entries=100, sequenceid=14, filesize=9.4 K 2024-11-20T12:15:01,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/21e1b508ae5f437f87875860b7022c4a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a 2024-11-20T12:15:01,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a, entries=100, sequenceid=14, filesize=9.4 K 2024-11-20T12:15:01,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 009722d6d4375a43f57af7eff44b236c in 291ms, sequenceid=14, compaction requested=false 2024-11-20T12:15:01,976 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T12:15:01,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:01,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-20T12:15:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:01,981 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:01,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T12:15:02,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207721baa6e98d4daeafc28f84f6c941c6_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104901775/Put/seqid=0 2024-11-20T12:15:02,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741955_1131 (size=12154) 2024-11-20T12:15:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:02,025 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207721baa6e98d4daeafc28f84f6c941c6_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207721baa6e98d4daeafc28f84f6c941c6_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:02,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1e66ac4310a744ed9e11dcf0339425a3, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:02,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1e66ac4310a744ed9e11dcf0339425a3 is 175, key is test_row_0/A:col10/1732104901775/Put/seqid=0 2024-11-20T12:15:02,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741956_1132 (size=30955) 2024-11-20T12:15:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:02,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:02,107 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:15:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104962117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104962121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104962127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104962133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104962151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104962257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104962258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104962258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104962258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104962276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T12:15:02,440 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1e66ac4310a744ed9e11dcf0339425a3 2024-11-20T12:15:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/9dc812944a69442884146fa5b81509e4 is 50, key is test_row_0/B:col10/1732104901775/Put/seqid=0 2024-11-20T12:15:02,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741957_1133 (size=12001) 2024-11-20T12:15:02,463 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/9dc812944a69442884146fa5b81509e4 2024-11-20T12:15:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104962467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104962467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104962467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0fc16fc1445847bf8e9f76af94d33355 is 50, key is test_row_0/C:col10/1732104901775/Put/seqid=0 2024-11-20T12:15:02,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741958_1134 (size=12001) 2024-11-20T12:15:02,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104962478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,484 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0fc16fc1445847bf8e9f76af94d33355 2024-11-20T12:15:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104962490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1e66ac4310a744ed9e11dcf0339425a3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3 2024-11-20T12:15:02,510 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3, entries=150, sequenceid=50, filesize=30.2 K 2024-11-20T12:15:02,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/9dc812944a69442884146fa5b81509e4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4 2024-11-20T12:15:02,521 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T12:15:02,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0fc16fc1445847bf8e9f76af94d33355 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355 2024-11-20T12:15:02,535 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355, entries=150, sequenceid=50, filesize=11.7 K 2024-11-20T12:15:02,537 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 009722d6d4375a43f57af7eff44b236c in 555ms, sequenceid=50, compaction requested=false 2024-11-20T12:15:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:02,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-20T12:15:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-20T12:15:02,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-20T12:15:02,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 868 msec 2024-11-20T12:15:02,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 886 msec 2024-11-20T12:15:02,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:02,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:02,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:02,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d41526e1c2d54766ad9062d71fdf86f8_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-20T12:15:02,807 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:02,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-20T12:15:02,810 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:02,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T12:15:02,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741959_1135 (size=14594) 2024-11-20T12:15:02,811 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:02,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:02,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:02,823 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d41526e1c2d54766ad9062d71fdf86f8_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d41526e1c2d54766ad9062d71fdf86f8_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:02,824 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/5ec8a96e7f7040639561c2520a88e2e3, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:02,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/5ec8a96e7f7040639561c2520a88e2e3 is 175, key is test_row_0/A:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:02,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104962819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732104902799, max timestamp=1732104902825Num uniques 859; 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732104902799, max timestamp=1732104902825Num uniques 859; 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104902799, max timestamp=1732104902825Num uniques 859; 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T12:15:02,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741960_1136 (size=39549) 2024-11-20T12:15:02,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/5ec8a96e7f7040639561c2520a88e2e3 2024-11-20T12:15:02,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d4a457b3e5884d98a2a72c0bca973eac is 50, key is test_row_0/B:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:02,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741961_1137 (size=12001) 2024-11-20T12:15:02,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104962860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104962861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104962869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104962873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T12:15:02,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104962937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T12:15:02,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:02,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:02,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:02,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:02,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104962976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104962987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104962987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104962987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:02,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T12:15:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:03,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T12:15:03,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104963147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104963187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104963197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104963197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104963197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d4a457b3e5884d98a2a72c0bca973eac 2024-11-20T12:15:03,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T12:15:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9f57045dfbfd4204bd0e3beade233d3c is 50, key is test_row_0/C:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:03,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:03,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741962_1138 (size=12001) 2024-11-20T12:15:03,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9f57045dfbfd4204bd0e3beade233d3c 2024-11-20T12:15:03,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/5ec8a96e7f7040639561c2520a88e2e3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3 2024-11-20T12:15:03,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3, entries=200, sequenceid=61, filesize=38.6 K 2024-11-20T12:15:03,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d4a457b3e5884d98a2a72c0bca973eac as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac 2024-11-20T12:15:03,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T12:15:03,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9f57045dfbfd4204bd0e3beade233d3c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c 2024-11-20T12:15:03,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T12:15:03,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 009722d6d4375a43f57af7eff44b236c in 551ms, sequenceid=61, compaction requested=true 2024-11-20T12:15:03,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:03,332 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:03,333 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:03,334 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:03,334 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,334 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=107.5 K 2024-11-20T12:15:03,334 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,334 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3] 2024-11-20T12:15:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:03,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:03,335 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:03,335 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06a3bd69263b4376b61a5ec6a01668c3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732104901674 2024-11-20T12:15:03,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:03,335 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e66ac4310a744ed9e11dcf0339425a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732104901752 2024-11-20T12:15:03,336 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:03,336 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:03,336 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,336 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=32.9 K 2024-11-20T12:15:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:03,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:03,337 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ec8a96e7f7040639561c2520a88e2e3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732104902106 2024-11-20T12:15:03,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b336c19735c74f309df8e4a286b2ddd7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732104901674 2024-11-20T12:15:03,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dc812944a69442884146fa5b81509e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732104901752 2024-11-20T12:15:03,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a457b3e5884d98a2a72c0bca973eac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732104902106 2024-11-20T12:15:03,347 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,352 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:03,353 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/40abecb1980d40559e915eb58a6030e0 is 50, key is test_row_0/B:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:03,355 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ae5d2cb76bb24de2a102e245f7634649_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,359 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ae5d2cb76bb24de2a102e245f7634649_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,360 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ae5d2cb76bb24de2a102e245f7634649_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741963_1139 (size=12104) 2024-11-20T12:15:03,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741964_1140 (size=4469) 2024-11-20T12:15:03,400 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#120 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:03,400 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/c94d5c0a708046e09db63445b0ff5a59 is 175, key is test_row_0/A:col10/1732104902106/Put/seqid=0 2024-11-20T12:15:03,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741965_1141 (size=31058) 2024-11-20T12:15:03,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T12:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,427 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T12:15:03,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T12:15:03,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112012086e8719264a14991da29383036249_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104902867/Put/seqid=0 2024-11-20T12:15:03,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741966_1142 (size=12154) 2024-11-20T12:15:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:03,454 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112012086e8719264a14991da29383036249_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012086e8719264a14991da29383036249_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3bb231aa9c2d444a88c7159ce2d0f765, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3bb231aa9c2d444a88c7159ce2d0f765 is 175, key is test_row_0/A:col10/1732104902867/Put/seqid=0 2024-11-20T12:15:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:03,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:03,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741967_1143 (size=30955) 2024-11-20T12:15:03,473 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3bb231aa9c2d444a88c7159ce2d0f765 2024-11-20T12:15:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/c1d1c2c6561b47c2ab931d7ea87a0f40 is 50, key is test_row_0/B:col10/1732104902867/Put/seqid=0 2024-11-20T12:15:03,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741968_1144 (size=12001) 2024-11-20T12:15:03,509 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/c1d1c2c6561b47c2ab931d7ea87a0f40 2024-11-20T12:15:03,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104963502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104963503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104963507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104963517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104963517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9b17fac427b14d85be41041af494c65b is 50, key is test_row_0/C:col10/1732104902867/Put/seqid=0 2024-11-20T12:15:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741969_1145 (size=12001) 2024-11-20T12:15:03,543 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9b17fac427b14d85be41041af494c65b 2024-11-20T12:15:03,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3bb231aa9c2d444a88c7159ce2d0f765 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765 2024-11-20T12:15:03,569 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765, entries=150, sequenceid=95, filesize=30.2 K 2024-11-20T12:15:03,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/c1d1c2c6561b47c2ab931d7ea87a0f40 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40 2024-11-20T12:15:03,580 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T12:15:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9b17fac427b14d85be41041af494c65b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b 2024-11-20T12:15:03,590 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b, entries=150, sequenceid=95, filesize=11.7 K 2024-11-20T12:15:03,591 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 009722d6d4375a43f57af7eff44b236c in 164ms, sequenceid=95, compaction requested=true 2024-11-20T12:15:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-20T12:15:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-20T12:15:03,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-20T12:15:03,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 781 msec 2024-11-20T12:15:03,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 788 msec 2024-11-20T12:15:03,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:03,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:03,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112016df4aa6e85c422ebac030774c594b42_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:03,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741970_1146 (size=17034) 2024-11-20T12:15:03,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:03,662 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112016df4aa6e85c422ebac030774c594b42_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016df4aa6e85c422ebac030774c594b42_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:03,666 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/671a1b2a47ff43499c377bd47c954b0b, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:03,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/671a1b2a47ff43499c377bd47c954b0b is 175, key is test_row_0/A:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:03,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741971_1147 (size=48139) 2024-11-20T12:15:03,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/671a1b2a47ff43499c377bd47c954b0b 2024-11-20T12:15:03,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/938bbb9e23524225b092262696ea6b33 is 50, key is test_row_0/B:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:03,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104963688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741972_1148 (size=12001) 2024-11-20T12:15:03,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/938bbb9e23524225b092262696ea6b33 2024-11-20T12:15:03,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104963690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/ff89aac1389f40ccabe07b5fdfb28954 is 50, key is test_row_0/C:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741973_1149 (size=12001) 2024-11-20T12:15:03,800 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/40abecb1980d40559e915eb58a6030e0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/40abecb1980d40559e915eb58a6030e0 2024-11-20T12:15:03,808 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 40abecb1980d40559e915eb58a6030e0(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:03,808 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:03,808 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104903334; duration=0sec 2024-11-20T12:15:03,808 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:03,808 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:03,808 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:03,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104963807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104963807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:03,814 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:03,814 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:03,814 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:03,814 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=44.6 K 2024-11-20T12:15:03,815 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e1b508ae5f437f87875860b7022c4a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732104901674 2024-11-20T12:15:03,816 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc16fc1445847bf8e9f76af94d33355, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732104901752 2024-11-20T12:15:03,816 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f57045dfbfd4204bd0e3beade233d3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732104902106 2024-11-20T12:15:03,816 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b17fac427b14d85be41041af494c65b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732104902857 2024-11-20T12:15:03,832 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/c94d5c0a708046e09db63445b0ff5a59 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59 2024-11-20T12:15:03,839 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:03,841 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/71dbe860d791408d8301e4859bf2ca16 is 50, key is test_row_0/C:col10/1732104902867/Put/seqid=0 2024-11-20T12:15:03,845 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into c94d5c0a708046e09db63445b0ff5a59(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:03,845 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:03,845 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104903332; duration=0sec 2024-11-20T12:15:03,845 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:03,845 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:03,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741974_1150 (size=12139) 2024-11-20T12:15:03,866 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/71dbe860d791408d8301e4859bf2ca16 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/71dbe860d791408d8301e4859bf2ca16 2024-11-20T12:15:03,875 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 71dbe860d791408d8301e4859bf2ca16(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:03,875 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:03,875 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=12, startTime=1732104903336; duration=0sec 2024-11-20T12:15:03,875 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:03,875 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T12:15:03,947 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:03,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T12:15:03,950 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:03,951 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:03,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T12:15:04,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104964016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104964017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104964017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104964027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104964027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T12:15:04,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T12:15:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:04,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:04,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:04,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/ff89aac1389f40ccabe07b5fdfb28954 2024-11-20T12:15:04,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/671a1b2a47ff43499c377bd47c954b0b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b 2024-11-20T12:15:04,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b, entries=250, sequenceid=108, filesize=47.0 K 2024-11-20T12:15:04,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/938bbb9e23524225b092262696ea6b33 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33 2024-11-20T12:15:04,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33, entries=150, sequenceid=108, filesize=11.7 K 2024-11-20T12:15:04,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/ff89aac1389f40ccabe07b5fdfb28954 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954 2024-11-20T12:15:04,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954, entries=150, sequenceid=108, filesize=11.7 K 2024-11-20T12:15:04,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 009722d6d4375a43f57af7eff44b236c in 559ms, sequenceid=108, compaction requested=true 2024-11-20T12:15:04,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:04,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:04,182 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:04,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:04,182 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:04,183 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:04,183 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:04,183 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,183 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=107.6 K 2024-11-20T12:15:04,183 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,184 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b] 2024-11-20T12:15:04,184 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:04,184 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:04,184 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,184 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/40abecb1980d40559e915eb58a6030e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=35.3 K 2024-11-20T12:15:04,184 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c94d5c0a708046e09db63445b0ff5a59, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732104902106 2024-11-20T12:15:04,185 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 40abecb1980d40559e915eb58a6030e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732104902106 2024-11-20T12:15:04,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:04,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:04,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:04,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:04,185 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bb231aa9c2d444a88c7159ce2d0f765, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732104902857 2024-11-20T12:15:04,185 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c1d1c2c6561b47c2ab931d7ea87a0f40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732104902857 2024-11-20T12:15:04,186 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 671a1b2a47ff43499c377bd47c954b0b, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732104903491 2024-11-20T12:15:04,187 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 938bbb9e23524225b092262696ea6b33, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732104903500 2024-11-20T12:15:04,207 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:04,207 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/94ae7778c73544159d9b3ee1071f0615 is 50, key is test_row_0/B:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:04,209 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:04,213 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ee0c0a7bb3804b9f84fe7aef5abca2c1_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:04,217 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ee0c0a7bb3804b9f84fe7aef5abca2c1_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:04,217 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ee0c0a7bb3804b9f84fe7aef5abca2c1_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:04,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741975_1151 (size=12207) 2024-11-20T12:15:04,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741976_1152 (size=4469) 2024-11-20T12:15:04,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,257 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:04,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T12:15:04,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0377a04ef95448397af7d18a954fb13_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104903672/Put/seqid=0 2024-11-20T12:15:04,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741977_1153 (size=12154) 2024-11-20T12:15:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:04,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104964355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104964357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104964466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104964467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T12:15:04,635 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#130 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:04,636 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/baf2f924bf904a8aadad418d1af09306 is 175, key is test_row_0/A:col10/1732104903500/Put/seqid=0 2024-11-20T12:15:04,640 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/94ae7778c73544159d9b3ee1071f0615 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/94ae7778c73544159d9b3ee1071f0615 2024-11-20T12:15:04,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741978_1154 (size=31161) 2024-11-20T12:15:04,648 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 94ae7778c73544159d9b3ee1071f0615(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:04,648 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:04,648 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104904182; duration=0sec 2024-11-20T12:15:04,648 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:04,648 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:04,648 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T12:15:04,653 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T12:15:04,653 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T12:15:04,653 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. because compaction request was cancelled 2024-11-20T12:15:04,653 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:04,658 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/baf2f924bf904a8aadad418d1af09306 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306 2024-11-20T12:15:04,667 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into baf2f924bf904a8aadad418d1af09306(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:04,668 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:04,668 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104904182; duration=0sec 2024-11-20T12:15:04,668 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:04,668 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:04,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104964677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104964677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:04,685 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0377a04ef95448397af7d18a954fb13_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0377a04ef95448397af7d18a954fb13_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:04,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/eeb1c6cde1e54f139f6a522ccc3fd391, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/eeb1c6cde1e54f139f6a522ccc3fd391 is 175, key is test_row_0/A:col10/1732104903672/Put/seqid=0 2024-11-20T12:15:04,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741979_1155 (size=30955) 2024-11-20T12:15:04,704 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/eeb1c6cde1e54f139f6a522ccc3fd391 2024-11-20T12:15:04,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6357260e361e437d9fa342eea0e88843 is 50, key is test_row_0/B:col10/1732104903672/Put/seqid=0 2024-11-20T12:15:04,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741980_1156 (size=12001) 2024-11-20T12:15:04,735 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6357260e361e437d9fa342eea0e88843 2024-11-20T12:15:04,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/defb2110686248a886118809c893fdf5 is 50, key is test_row_0/C:col10/1732104903672/Put/seqid=0 2024-11-20T12:15:04,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741981_1157 (size=12001) 2024-11-20T12:15:04,764 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/defb2110686248a886118809c893fdf5 2024-11-20T12:15:04,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/eeb1c6cde1e54f139f6a522ccc3fd391 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391 2024-11-20T12:15:04,778 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391, entries=150, sequenceid=130, filesize=30.2 K 2024-11-20T12:15:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6357260e361e437d9fa342eea0e88843 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843 2024-11-20T12:15:04,786 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T12:15:04,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/defb2110686248a886118809c893fdf5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5 2024-11-20T12:15:04,794 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5, entries=150, sequenceid=130, filesize=11.7 K 2024-11-20T12:15:04,795 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 009722d6d4375a43f57af7eff44b236c in 537ms, sequenceid=130, compaction requested=true 2024-11-20T12:15:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:04,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T12:15:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T12:15:04,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T12:15:04,798 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 845 msec 2024-11-20T12:15:04,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 853 msec 2024-11-20T12:15:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:04,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:04,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:05,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112071b42f0af2c94387a7a9f697b1772578_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:05,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741982_1158 (size=17284) 2024-11-20T12:15:05,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:05,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104965029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104965032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112071b42f0af2c94387a7a9f697b1772578_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112071b42f0af2c94387a7a9f697b1772578_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:05,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ab9b3850f88a47b8b53fbfd7f047dacf, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ab9b3850f88a47b8b53fbfd7f047dacf is 175, key is test_row_0/A:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:05,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104965036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741983_1159 (size=48389) 2024-11-20T12:15:05,045 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ab9b3850f88a47b8b53fbfd7f047dacf 2024-11-20T12:15:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104965037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104965046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,050 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,050 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/769179bdfed7472d8c1d0f83debbe4d8 is 50, key is test_row_0/B:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741984_1160 (size=12151) 2024-11-20T12:15:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T12:15:05,087 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:05,088 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T12:15:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:05,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:05,092 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:05,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:05,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104965146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104965146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104965146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:05,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T12:15:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104965356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104965357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104965357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T12:15:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:05,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/769179bdfed7472d8c1d0f83debbe4d8 2024-11-20T12:15:05,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/91b6e977a890414599deb8b905691e4e is 50, key is test_row_0/C:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:05,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741985_1161 (size=12151) 2024-11-20T12:15:05,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/91b6e977a890414599deb8b905691e4e 2024-11-20T12:15:05,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ab9b3850f88a47b8b53fbfd7f047dacf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf 2024-11-20T12:15:05,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf, entries=250, sequenceid=147, filesize=47.3 K 2024-11-20T12:15:05,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/769179bdfed7472d8c1d0f83debbe4d8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8 2024-11-20T12:15:05,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8, entries=150, sequenceid=147, filesize=11.9 K 2024-11-20T12:15:05,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/91b6e977a890414599deb8b905691e4e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e 2024-11-20T12:15:05,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e, entries=150, sequenceid=147, filesize=11.9 K 2024-11-20T12:15:05,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 009722d6d4375a43f57af7eff44b236c in 550ms, sequenceid=147, compaction requested=true 2024-11-20T12:15:05,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:05,542 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:05,543 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:05,544 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:05,544 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,544 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=107.9 K 2024-11-20T12:15:05,544 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,544 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf] 2024-11-20T12:15:05,545 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting baf2f924bf904a8aadad418d1af09306, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732104903500 2024-11-20T12:15:05,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:05,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:05,545 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb1c6cde1e54f139f6a522ccc3fd391, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732104903672 2024-11-20T12:15:05,545 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9b3850f88a47b8b53fbfd7f047dacf, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:05,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:05,549 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:05,549 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:05,549 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,549 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/94ae7778c73544159d9b3ee1071f0615, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=35.5 K 2024-11-20T12:15:05,550 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 94ae7778c73544159d9b3ee1071f0615, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732104903500 2024-11-20T12:15:05,550 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6357260e361e437d9fa342eea0e88843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732104903672 2024-11-20T12:15:05,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:05,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:05,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:05,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 769179bdfed7472d8c1d0f83debbe4d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:05,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:05,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T12:15:05,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:05,556 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:05,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:05,561 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,577 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207524f2e61e03462eb4722d3ec125adf2_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,580 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207524f2e61e03462eb4722d3ec125adf2_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,580 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207524f2e61e03462eb4722d3ec125adf2_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,591 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:05,591 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/037344539f3145a08448b77cc2b5cd15 is 50, key is test_row_0/B:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:05,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200c7cec672d554a64a14397ab5335a4e3_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104905029/Put/seqid=0 2024-11-20T12:15:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:05,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:05,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741986_1162 (size=4469) 2024-11-20T12:15:05,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741987_1163 (size=12459) 2024-11-20T12:15:05,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104965689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104965690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104965695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741988_1164 (size=12304) 2024-11-20T12:15:05,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:05,711 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200c7cec672d554a64a14397ab5335a4e3_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c7cec672d554a64a14397ab5335a4e3_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:05,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ae7aeef66f84a289d6237f5c8e037e2, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:05,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ae7aeef66f84a289d6237f5c8e037e2 is 175, key is test_row_0/A:col10/1732104905029/Put/seqid=0 2024-11-20T12:15:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:05,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741989_1165 (size=31105) 2024-11-20T12:15:05,736 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ae7aeef66f84a289d6237f5c8e037e2 2024-11-20T12:15:05,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/ba8f4dbca8e94307a1443e682c252e3c is 50, key is test_row_0/B:col10/1732104905029/Put/seqid=0 2024-11-20T12:15:05,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741990_1166 (size=12151) 2024-11-20T12:15:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104965807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104965807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104965807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:05,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104966016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104966016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104966027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,083 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#137 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:06,084 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/d5ee60a4d9df421db216a7e21eed4194 is 175, key is test_row_0/A:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:06,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741991_1167 (size=31413) 2024-11-20T12:15:06,100 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/037344539f3145a08448b77cc2b5cd15 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/037344539f3145a08448b77cc2b5cd15 2024-11-20T12:15:06,105 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 037344539f3145a08448b77cc2b5cd15(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:06,106 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:06,106 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104905547; duration=0sec 2024-11-20T12:15:06,106 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:06,106 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:06,106 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:06,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:06,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:06,109 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:06,109 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/71dbe860d791408d8301e4859bf2ca16, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=47.2 K 2024-11-20T12:15:06,109 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 71dbe860d791408d8301e4859bf2ca16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732104902857 2024-11-20T12:15:06,110 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ff89aac1389f40ccabe07b5fdfb28954, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732104903500 2024-11-20T12:15:06,110 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting defb2110686248a886118809c893fdf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732104903672 2024-11-20T12:15:06,110 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 91b6e977a890414599deb8b905691e4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:06,119 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:06,120 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/ae81a35d7b5a4b1b935a2b156688be0e is 50, key is test_row_0/C:col10/1732104904340/Put/seqid=0 2024-11-20T12:15:06,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741992_1168 (size=12425) 2024-11-20T12:15:06,136 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/ae81a35d7b5a4b1b935a2b156688be0e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ae81a35d7b5a4b1b935a2b156688be0e 2024-11-20T12:15:06,141 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into ae81a35d7b5a4b1b935a2b156688be0e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:06,141 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:06,141 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=12, startTime=1732104905551; duration=0sec 2024-11-20T12:15:06,142 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:06,142 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:06,154 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/ba8f4dbca8e94307a1443e682c252e3c 2024-11-20T12:15:06,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/c8e761f275c845b7a9e14010f791d8a3 is 50, key is test_row_0/C:col10/1732104905029/Put/seqid=0 2024-11-20T12:15:06,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741993_1169 (size=12151) 2024-11-20T12:15:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104966326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104966326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104966336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,495 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/d5ee60a4d9df421db216a7e21eed4194 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194 2024-11-20T12:15:06,502 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into d5ee60a4d9df421db216a7e21eed4194(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:06,502 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:06,502 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104905542; duration=0sec 2024-11-20T12:15:06,502 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:06,502 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:06,566 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/c8e761f275c845b7a9e14010f791d8a3 2024-11-20T12:15:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ae7aeef66f84a289d6237f5c8e037e2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2 2024-11-20T12:15:06,582 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2, entries=150, sequenceid=164, filesize=30.4 K 2024-11-20T12:15:06,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/ba8f4dbca8e94307a1443e682c252e3c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c 2024-11-20T12:15:06,588 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T12:15:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/c8e761f275c845b7a9e14010f791d8a3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3 2024-11-20T12:15:06,600 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3, entries=150, sequenceid=164, filesize=11.9 K 2024-11-20T12:15:06,601 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 009722d6d4375a43f57af7eff44b236c in 1045ms, sequenceid=164, compaction requested=false 2024-11-20T12:15:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T12:15:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T12:15:06,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T12:15:06,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-20T12:15:06,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5200 sec 2024-11-20T12:15:06,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:06,839 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:06,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:06,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a118e9b2832a4c1aafd3cb8a0579c66d_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:06,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741994_1170 (size=17284) 2024-11-20T12:15:06,867 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:06,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a118e9b2832a4c1aafd3cb8a0579c66d_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a118e9b2832a4c1aafd3cb8a0579c66d_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104966860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85b43ab7b462408a9300dc66bac058d5, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:06,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85b43ab7b462408a9300dc66bac058d5 is 175, key is test_row_0/A:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:06,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104966873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104966873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741995_1171 (size=48389) 2024-11-20T12:15:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104966987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104966987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104966987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104967057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,059 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104967067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,071 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,077 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:15:07,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104967196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104967196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104967196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T12:15:07,237 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:07,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T12:15:07,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:07,240 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:07,240 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:07,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:07,298 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85b43ab7b462408a9300dc66bac058d5 2024-11-20T12:15:07,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/aa9f5091189a4c3c89385f18eec6a844 is 50, key is test_row_0/B:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:07,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741996_1172 (size=12151) 2024-11-20T12:15:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:07,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104967506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104967506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104967506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:07,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:07,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:07,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:07,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/aa9f5091189a4c3c89385f18eec6a844 2024-11-20T12:15:07,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/048fc888e7164a25b726c02ea6613090 is 50, key is test_row_0/C:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:07,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741997_1173 (size=12151) 2024-11-20T12:15:07,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:07,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:07,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:08,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:08,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:08,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104968017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:08,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:08,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104968017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104968018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:08,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/048fc888e7164a25b726c02ea6613090 2024-11-20T12:15:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85b43ab7b462408a9300dc66bac058d5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5 2024-11-20T12:15:08,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5, entries=250, sequenceid=184, filesize=47.3 K 2024-11-20T12:15:08,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/aa9f5091189a4c3c89385f18eec6a844 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844 2024-11-20T12:15:08,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844, entries=150, sequenceid=184, filesize=11.9 K 2024-11-20T12:15:08,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/048fc888e7164a25b726c02ea6613090 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090 2024-11-20T12:15:08,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:08,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:08,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090, entries=150, sequenceid=184, filesize=11.9 K 2024-11-20T12:15:08,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 009722d6d4375a43f57af7eff44b236c in 1332ms, sequenceid=184, compaction requested=true 2024-11-20T12:15:08,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:08,172 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:08,172 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:08,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:08,181 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:08,181 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:08,181 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,181 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/037344539f3145a08448b77cc2b5cd15, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=35.9 K 2024-11-20T12:15:08,182 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:08,182 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:08,182 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 037344539f3145a08448b77cc2b5cd15, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:08,182 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,182 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=108.3 K 2024-11-20T12:15:08,182 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,182 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5] 2024-11-20T12:15:08,183 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d5ee60a4d9df421db216a7e21eed4194, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:08,183 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba8f4dbca8e94307a1443e682c252e3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732104905015 2024-11-20T12:15:08,183 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae7aeef66f84a289d6237f5c8e037e2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732104905015 2024-11-20T12:15:08,183 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa9f5091189a4c3c89385f18eec6a844, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:08,184 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 85b43ab7b462408a9300dc66bac058d5, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:08,203 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:08,222 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:08,222 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112022261a1ba9cf4da6b064306a0eb360dd_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:08,223 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8bc492c71a6243af8dec19d8311f9e6f is 50, key is test_row_0/B:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:08,225 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112022261a1ba9cf4da6b064306a0eb360dd_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:08,225 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112022261a1ba9cf4da6b064306a0eb360dd_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:08,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741998_1174 (size=12561) 2024-11-20T12:15:08,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741999_1175 (size=4469) 2024-11-20T12:15:08,290 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8bc492c71a6243af8dec19d8311f9e6f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8bc492c71a6243af8dec19d8311f9e6f 2024-11-20T12:15:08,291 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#146 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:08,292 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85e2619803aa459ebed97b3e54a0e978 is 175, key is test_row_0/A:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:08,304 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 8bc492c71a6243af8dec19d8311f9e6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:08,304 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:08,304 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104908172; duration=0sec 2024-11-20T12:15:08,304 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:08,304 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:08,304 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:08,308 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:08,308 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:08,308 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,308 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ae81a35d7b5a4b1b935a2b156688be0e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=35.9 K 2024-11-20T12:15:08,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742000_1176 (size=31515) 2024-11-20T12:15:08,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae81a35d7b5a4b1b935a2b156688be0e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732104904340 2024-11-20T12:15:08,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e761f275c845b7a9e14010f791d8a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732104905015 2024-11-20T12:15:08,314 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 048fc888e7164a25b726c02ea6613090, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:08,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T12:15:08,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:08,338 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:08,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:08,364 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:08,364 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/3731dbe8013445ef818d06170f841f01 is 50, key is test_row_0/C:col10/1732104905677/Put/seqid=0 2024-11-20T12:15:08,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bd47662fc99b43f295d0bbffa1fa1133_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104906869/Put/seqid=0 2024-11-20T12:15:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:08,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742001_1177 (size=12527) 2024-11-20T12:15:08,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742002_1178 (size=12304) 2024-11-20T12:15:08,714 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/85e2619803aa459ebed97b3e54a0e978 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978 2024-11-20T12:15:08,720 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into 85e2619803aa459ebed97b3e54a0e978(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:08,720 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:08,720 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104908172; duration=0sec 2024-11-20T12:15:08,720 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:08,720 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:08,787 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/3731dbe8013445ef818d06170f841f01 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/3731dbe8013445ef818d06170f841f01 2024-11-20T12:15:08,792 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 3731dbe8013445ef818d06170f841f01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:08,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:08,792 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=13, startTime=1732104908172; duration=0sec 2024-11-20T12:15:08,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:08,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:08,797 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bd47662fc99b43f295d0bbffa1fa1133_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd47662fc99b43f295d0bbffa1fa1133_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:08,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/26a3f68e25c64e31af31df8c1714bb65, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:08,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/26a3f68e25c64e31af31df8c1714bb65 is 175, key is test_row_0/A:col10/1732104906869/Put/seqid=0 2024-11-20T12:15:08,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742003_1179 (size=31105) 2024-11-20T12:15:08,803 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/26a3f68e25c64e31af31df8c1714bb65 2024-11-20T12:15:08,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d7d0295d1a4e405191d8feeaf402e4ab is 50, key is test_row_0/B:col10/1732104906869/Put/seqid=0 2024-11-20T12:15:08,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742004_1180 (size=12151) 2024-11-20T12:15:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:09,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:09,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104969060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104969061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104969062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104969166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104969176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104969177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,225 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d7d0295d1a4e405191d8feeaf402e4ab 2024-11-20T12:15:09,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6672f56186f949eda949bb94566ddf8c is 50, key is test_row_0/C:col10/1732104906869/Put/seqid=0 2024-11-20T12:15:09,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742005_1181 (size=12151) 2024-11-20T12:15:09,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104969376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104969386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104969386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,638 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6672f56186f949eda949bb94566ddf8c 2024-11-20T12:15:09,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/26a3f68e25c64e31af31df8c1714bb65 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65 2024-11-20T12:15:09,646 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65, entries=150, sequenceid=200, filesize=30.4 K 2024-11-20T12:15:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/d7d0295d1a4e405191d8feeaf402e4ab as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab 2024-11-20T12:15:09,651 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab, entries=150, sequenceid=200, filesize=11.9 K 2024-11-20T12:15:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6672f56186f949eda949bb94566ddf8c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c 2024-11-20T12:15:09,658 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-20T12:15:09,658 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 009722d6d4375a43f57af7eff44b236c in 1321ms, sequenceid=200, compaction requested=false 2024-11-20T12:15:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:09,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T12:15:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T12:15:09,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T12:15:09,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4200 sec 2024-11-20T12:15:09,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.4240 sec 2024-11-20T12:15:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:09,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T12:15:09,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:09,692 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:09,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203debf88fd4134d369329a75945c3920e_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:09,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742006_1182 (size=14794) 2024-11-20T12:15:09,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104969711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104969712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104969714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104969816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104969826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,830 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104969827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104970026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104970036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104970036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:10,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203debf88fd4134d369329a75945c3920e_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203debf88fd4134d369329a75945c3920e_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:10,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0b4bceb5e7d24ef7b1a35328ca7b1f30, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 is 175, key is test_row_0/A:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742007_1183 (size=39749) 2024-11-20T12:15:10,121 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 2024-11-20T12:15:10,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/612a2a9aa7664451ab464107a04cbb00 is 50, key is test_row_0/B:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742008_1184 (size=12151) 2024-11-20T12:15:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104970337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104970347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104970347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,517 INFO [master/843d83d51ba0:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T12:15:10,517 INFO [master/843d83d51ba0:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T12:15:10,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/612a2a9aa7664451ab464107a04cbb00 2024-11-20T12:15:10,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/d37ea0c3d2c54e47a2227afdcc994e7d is 50, key is test_row_0/C:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:10,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742009_1185 (size=12151) 2024-11-20T12:15:10,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/d37ea0c3d2c54e47a2227afdcc994e7d 2024-11-20T12:15:10,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 2024-11-20T12:15:10,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30, entries=200, sequenceid=220, filesize=38.8 K 2024-11-20T12:15:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/612a2a9aa7664451ab464107a04cbb00 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00 2024-11-20T12:15:10,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00, entries=150, sequenceid=220, filesize=11.9 K 2024-11-20T12:15:10,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/d37ea0c3d2c54e47a2227afdcc994e7d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d 2024-11-20T12:15:10,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d, entries=150, sequenceid=220, filesize=11.9 K 2024-11-20T12:15:10,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 009722d6d4375a43f57af7eff44b236c in 882ms, sequenceid=220, compaction requested=true 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:10,573 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:10,573 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:10,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:10,574 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:10,574 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:10,574 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:10,574 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:10,575 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:10,575 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:10,575 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=100.0 K 2024-11-20T12:15:10,575 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8bc492c71a6243af8dec19d8311f9e6f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.0 K 2024-11-20T12:15:10,575 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:10,575 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30] 2024-11-20T12:15:10,575 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85e2619803aa459ebed97b3e54a0e978, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:10,575 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bc492c71a6243af8dec19d8311f9e6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:10,575 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a3f68e25c64e31af31df8c1714bb65, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732104906858 2024-11-20T12:15:10,575 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d7d0295d1a4e405191d8feeaf402e4ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732104906858 2024-11-20T12:15:10,576 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b4bceb5e7d24ef7b1a35328ca7b1f30, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:10,576 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 612a2a9aa7664451ab464107a04cbb00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:10,640 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,642 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:10,642 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/cc509e282442442195fd6c24677443b4 is 50, key is test_row_0/B:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:10,643 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b5ea4eebee73441bb619cce289d1b44c_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,645 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b5ea4eebee73441bb619cce289d1b44c_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,645 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5ea4eebee73441bb619cce289d1b44c_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742010_1186 (size=12663) 2024-11-20T12:15:10,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742011_1187 (size=4469) 2024-11-20T12:15:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:10,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T12:15:10,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:10,850 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:10,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af04a223965d48fa9364271650573d0b_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104910848/Put/seqid=0 2024-11-20T12:15:10,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742012_1188 (size=14794) 2024-11-20T12:15:10,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:10,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120af04a223965d48fa9364271650573d0b_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af04a223965d48fa9364271650573d0b_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:10,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ed367f49dff4d5ebce0e980d888a33c, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:10,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ed367f49dff4d5ebce0e980d888a33c is 175, key is test_row_0/A:col10/1732104910848/Put/seqid=0 2024-11-20T12:15:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742013_1189 (size=39749) 2024-11-20T12:15:10,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104970878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104970880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104970881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104970986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104970986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104970986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:10,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:10,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,050 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:11,051 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/80b5feabe3794978832c65d655578b2c is 175, key is test_row_0/A:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:11,055 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/cc509e282442442195fd6c24677443b4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/cc509e282442442195fd6c24677443b4 2024-11-20T12:15:11,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742014_1190 (size=31617) 2024-11-20T12:15:11,061 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into cc509e282442442195fd6c24677443b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:11,061 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:11,061 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104910573; duration=0sec 2024-11-20T12:15:11,062 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:11,062 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:11,062 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:11,062 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/80b5feabe3794978832c65d655578b2c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c 2024-11-20T12:15:11,065 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:11,065 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:11,065 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,065 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/3731dbe8013445ef818d06170f841f01, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.0 K 2024-11-20T12:15:11,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3731dbe8013445ef818d06170f841f01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732104905677 2024-11-20T12:15:11,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6672f56186f949eda949bb94566ddf8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732104906858 2024-11-20T12:15:11,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d37ea0c3d2c54e47a2227afdcc994e7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:11,069 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into 80b5feabe3794978832c65d655578b2c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:11,069 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:11,069 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104910573; duration=0sec 2024-11-20T12:15:11,069 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:11,069 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:11,075 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:11,075 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/8e595f6efea747d9852cc36f400f1750 is 50, key is test_row_0/C:col10/1732104909042/Put/seqid=0 2024-11-20T12:15:11,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742015_1191 (size=12629) 2024-11-20T12:15:11,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104971076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,082 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,087 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/8e595f6efea747d9852cc36f400f1750 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/8e595f6efea747d9852cc36f400f1750 2024-11-20T12:15:11,097 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 8e595f6efea747d9852cc36f400f1750(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:11,097 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:11,097 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=13, startTime=1732104910573; duration=0sec 2024-11-20T12:15:11,098 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:11,098 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:11,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104971097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,099 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104971197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104971197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104971197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ed367f49dff4d5ebce0e980d888a33c 2024-11-20T12:15:11,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4dbe3f0381af428a868bff7a001e27ab is 50, key is test_row_0/B:col10/1732104910848/Put/seqid=0 2024-11-20T12:15:11,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742016_1192 (size=12151) 2024-11-20T12:15:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T12:15:11,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:11,398 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T12:15:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:11,400 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:11,401 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:11,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104971507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104971507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:11,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104971516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:11,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:11,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T12:15:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4dbe3f0381af428a868bff7a001e27ab 2024-11-20T12:15:11,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0f5f893e684f43c997e2863ae7a45355 is 50, key is test_row_0/C:col10/1732104910848/Put/seqid=0 2024-11-20T12:15:11,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T12:15:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742017_1193 (size=12151) 2024-11-20T12:15:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:11,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T12:15:11,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:11,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:11,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:12,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T12:15:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:12,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:12,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104972017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:12,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:12,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104972017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104972026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:12,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0f5f893e684f43c997e2863ae7a45355 2024-11-20T12:15:12,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4ed367f49dff4d5ebce0e980d888a33c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c 2024-11-20T12:15:12,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c, entries=200, sequenceid=235, filesize=38.8 K 2024-11-20T12:15:12,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4dbe3f0381af428a868bff7a001e27ab as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab 2024-11-20T12:15:12,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T12:15:12,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/0f5f893e684f43c997e2863ae7a45355 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355 2024-11-20T12:15:12,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355, entries=150, sequenceid=235, filesize=11.9 K 2024-11-20T12:15:12,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 009722d6d4375a43f57af7eff44b236c in 1280ms, sequenceid=235, compaction requested=false 2024-11-20T12:15:12,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:12,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T12:15:12,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:12,165 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:12,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:12,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203ad69d861d5341559b93ab76a5c46ba6_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104910879/Put/seqid=0 2024-11-20T12:15:12,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742018_1194 (size=12304) 2024-11-20T12:15:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:12,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:12,581 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203ad69d861d5341559b93ab76a5c46ba6_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ad69d861d5341559b93ab76a5c46ba6_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1093d551e4a84805973677ed97e42938, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1093d551e4a84805973677ed97e42938 is 175, key is test_row_0/A:col10/1732104910879/Put/seqid=0 2024-11-20T12:15:12,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742019_1195 (size=31105) 2024-11-20T12:15:12,989 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1093d551e4a84805973677ed97e42938 2024-11-20T12:15:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8c736103d7ae4689a7b657af58ddffca is 50, key is test_row_0/B:col10/1732104910879/Put/seqid=0 2024-11-20T12:15:13,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742020_1196 (size=12151) 2024-11-20T12:15:13,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:13,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:13,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104973047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104973052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104973053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,058 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104973157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104973166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104973166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104973367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104973377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104973377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,402 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8c736103d7ae4689a7b657af58ddffca 2024-11-20T12:15:13,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/fe7d6bdb64b8448b85bf202f8af8ba4b is 50, key is test_row_0/C:col10/1732104910879/Put/seqid=0 2024-11-20T12:15:13,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742021_1197 (size=12151) 2024-11-20T12:15:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104973676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104973687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104973687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:13,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:13,813 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/fe7d6bdb64b8448b85bf202f8af8ba4b 2024-11-20T12:15:13,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/1093d551e4a84805973677ed97e42938 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938 2024-11-20T12:15:13,821 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938, entries=150, sequenceid=255, filesize=30.4 K 2024-11-20T12:15:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8c736103d7ae4689a7b657af58ddffca as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca 2024-11-20T12:15:13,826 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca, entries=150, sequenceid=255, filesize=11.9 K 2024-11-20T12:15:13,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/fe7d6bdb64b8448b85bf202f8af8ba4b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b 2024-11-20T12:15:13,830 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b, entries=150, sequenceid=255, filesize=11.9 K 2024-11-20T12:15:13,831 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 009722d6d4375a43f57af7eff44b236c in 1667ms, sequenceid=255, compaction requested=true 2024-11-20T12:15:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:13,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:13,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T12:15:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T12:15:13,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T12:15:13,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4320 sec 2024-11-20T12:15:13,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4370 sec 2024-11-20T12:15:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:14,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:14,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:14,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202a0eff66b6e443bb908be936f6bece75_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:14,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742022_1198 (size=14994) 2024-11-20T12:15:14,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104974212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104974213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104974214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104974327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104974327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104974327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104974537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104974537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104974537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,601 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:14,606 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202a0eff66b6e443bb908be936f6bece75_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a0eff66b6e443bb908be936f6bece75_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:14,607 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4602c7163e304f35befeba79b01423f8, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:14,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4602c7163e304f35befeba79b01423f8 is 175, key is test_row_0/A:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:14,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742023_1199 (size=39949) 2024-11-20T12:15:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104974846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104974846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:14,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:14,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104974847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:15,012 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4602c7163e304f35befeba79b01423f8 2024-11-20T12:15:15,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8f8e99aae6e34b918fe6d23f374c9dbf is 50, key is test_row_0/B:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:15,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742024_1200 (size=12301) 2024-11-20T12:15:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104975356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:15,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104975357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104975357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:15,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:15,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8f8e99aae6e34b918fe6d23f374c9dbf 2024-11-20T12:15:15,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/afe59aa6b8fd4dd2ab22f03b932d568c is 50, key is test_row_0/C:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:15,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742025_1201 (size=12301) 2024-11-20T12:15:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T12:15:15,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:15,567 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T12:15:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T12:15:15,569 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:15,570 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:15,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T12:15:15,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T12:15:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:15,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:15,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/afe59aa6b8fd4dd2ab22f03b932d568c 2024-11-20T12:15:15,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4602c7163e304f35befeba79b01423f8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8 2024-11-20T12:15:15,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8, entries=200, sequenceid=271, filesize=39.0 K 2024-11-20T12:15:15,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8f8e99aae6e34b918fe6d23f374c9dbf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf 2024-11-20T12:15:15,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf, entries=150, sequenceid=271, filesize=12.0 K 2024-11-20T12:15:15,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/afe59aa6b8fd4dd2ab22f03b932d568c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c 2024-11-20T12:15:15,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c, entries=150, sequenceid=271, filesize=12.0 K 2024-11-20T12:15:15,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 009722d6d4375a43f57af7eff44b236c in 1665ms, sequenceid=271, compaction requested=true 2024-11-20T12:15:15,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:15,855 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:15,855 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:15,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:15,856 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,856 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,856 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/cc509e282442442195fd6c24677443b4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=48.1 K 2024-11-20T12:15:15,856 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=139.1 K 2024-11-20T12:15:15,856 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8] 2024-11-20T12:15:15,856 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cc509e282442442195fd6c24677443b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:15,857 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80b5feabe3794978832c65d655578b2c, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:15,857 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dbe3f0381af428a868bff7a001e27ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732104909709 2024-11-20T12:15:15,857 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ed367f49dff4d5ebce0e980d888a33c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732104909709 2024-11-20T12:15:15,857 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c736103d7ae4689a7b657af58ddffca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732104910860 2024-11-20T12:15:15,857 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1093d551e4a84805973677ed97e42938, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732104910860 2024-11-20T12:15:15,858 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8e99aae6e34b918fe6d23f374c9dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913050 2024-11-20T12:15:15,858 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4602c7163e304f35befeba79b01423f8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913046 2024-11-20T12:15:15,867 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:15,867 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/9dcc35aadbf34679b03f0ba271bd6729 is 50, key is test_row_0/B:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:15,874 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:15,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,876 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:15,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742026_1202 (size=12949) 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:15,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:15,881 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202cbbd5a44b8443539b64dd58eb759053_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:15,883 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202cbbd5a44b8443539b64dd58eb759053_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:15,884 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202cbbd5a44b8443539b64dd58eb759053_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:15,886 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/9dcc35aadbf34679b03f0ba271bd6729 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dcc35aadbf34679b03f0ba271bd6729 2024-11-20T12:15:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T12:15:15,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073e0a9759e204e3aa1c8703370130d68_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104914209/Put/seqid=0 2024-11-20T12:15:15,897 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 9dcc35aadbf34679b03f0ba271bd6729(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:15,897 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:15,897 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=12, startTime=1732104915855; duration=0sec 2024-11-20T12:15:15,897 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:15,897 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:15,898 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:15,902 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:15,902 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:15,902 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:15,902 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/8e595f6efea747d9852cc36f400f1750, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=48.1 K 2024-11-20T12:15:15,903 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e595f6efea747d9852cc36f400f1750, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1732104909042 2024-11-20T12:15:15,903 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f5f893e684f43c997e2863ae7a45355, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732104909709 2024-11-20T12:15:15,904 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7d6bdb64b8448b85bf202f8af8ba4b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732104910860 2024-11-20T12:15:15,904 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting afe59aa6b8fd4dd2ab22f03b932d568c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913050 2024-11-20T12:15:15,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742027_1203 (size=4469) 2024-11-20T12:15:15,918 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#168 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:15,918 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/53c61abf5c9e486cbafc29e70ca80ac4 is 175, key is test_row_0/A:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:15,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742028_1204 (size=12454) 2024-11-20T12:15:15,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:15,925 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112073e0a9759e204e3aa1c8703370130d68_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073e0a9759e204e3aa1c8703370130d68_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3a5850a2a0cb415f94dc9b22a89fe746, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:15,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3a5850a2a0cb415f94dc9b22a89fe746 is 175, key is test_row_0/A:col10/1732104914209/Put/seqid=0 2024-11-20T12:15:15,945 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:15,945 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/862aad6e33da4a53bfe686cb536be02e is 50, key is test_row_0/C:col10/1732104913051/Put/seqid=0 2024-11-20T12:15:15,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742029_1205 (size=31903) 2024-11-20T12:15:15,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742030_1206 (size=31255) 2024-11-20T12:15:15,956 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3a5850a2a0cb415f94dc9b22a89fe746 2024-11-20T12:15:15,957 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/53c61abf5c9e486cbafc29e70ca80ac4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4 2024-11-20T12:15:15,969 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into 53c61abf5c9e486cbafc29e70ca80ac4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:15,969 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:15,969 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=12, startTime=1732104915854; duration=0sec 2024-11-20T12:15:15,969 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:15,969 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:15,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742031_1207 (size=12915) 2024-11-20T12:15:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/3e3678a085c14b9c8c1816c45377a90b is 50, key is test_row_0/B:col10/1732104914209/Put/seqid=0 2024-11-20T12:15:15,978 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/862aad6e33da4a53bfe686cb536be02e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/862aad6e33da4a53bfe686cb536be02e 2024-11-20T12:15:15,986 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 862aad6e33da4a53bfe686cb536be02e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:15,987 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:15,987 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=12, startTime=1732104915855; duration=0sec 2024-11-20T12:15:15,987 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:15,987 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:15,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742032_1208 (size=12301) 2024-11-20T12:15:15,991 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/3e3678a085c14b9c8c1816c45377a90b 2024-11-20T12:15:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6eff2b29df0d4eef8296c5c52d4b535f is 50, key is test_row_0/C:col10/1732104914209/Put/seqid=0 2024-11-20T12:15:16,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742033_1209 (size=12301) 2024-11-20T12:15:16,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T12:15:16,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104976404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104976406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104976407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,414 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6eff2b29df0d4eef8296c5c52d4b535f 2024-11-20T12:15:16,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/3a5850a2a0cb415f94dc9b22a89fe746 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746 2024-11-20T12:15:16,425 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746, entries=150, sequenceid=287, filesize=30.5 K 2024-11-20T12:15:16,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/3e3678a085c14b9c8c1816c45377a90b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b 2024-11-20T12:15:16,432 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T12:15:16,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/6eff2b29df0d4eef8296c5c52d4b535f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f 2024-11-20T12:15:16,438 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T12:15:16,439 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 009722d6d4375a43f57af7eff44b236c in 563ms, sequenceid=287, compaction requested=false 2024-11-20T12:15:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T12:15:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T12:15:16,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T12:15:16,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 870 msec 2024-11-20T12:15:16,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 876 msec 2024-11-20T12:15:16,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:16,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:16,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:16,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120086cfa6a1c184d66a622d3ec318a8684_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:16,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742034_1210 (size=12454) 2024-11-20T12:15:16,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104976546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104976548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104976549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104976656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104976656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104976656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T12:15:16,707 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:16,708 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T12:15:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:16,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:16,710 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:16,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:16,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:16,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:16,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104976866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104976866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104976866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:16,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:16,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120086cfa6a1c184d66a622d3ec318a8684_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120086cfa6a1c184d66a622d3ec318a8684_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:16,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/272c6375255349a2a8b17ee6bbe741ca, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:16,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/272c6375255349a2a8b17ee6bbe741ca is 175, key is test_row_0/A:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:16,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742035_1211 (size=31255) 2024-11-20T12:15:16,947 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/272c6375255349a2a8b17ee6bbe741ca 2024-11-20T12:15:16,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/19b7da0135e84a67881bcb9a524c3091 is 50, key is test_row_0/B:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:16,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742036_1212 (size=12301) 2024-11-20T12:15:17,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:17,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104977177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104977177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104977177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:17,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/19b7da0135e84a67881bcb9a524c3091 2024-11-20T12:15:17,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb14548b7754ec38ff9963a9f9b94e4 is 50, key is test_row_0/C:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:17,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742037_1213 (size=12301) 2024-11-20T12:15:17,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104977687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104977687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104977687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:17,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb14548b7754ec38ff9963a9f9b94e4 2024-11-20T12:15:17,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/272c6375255349a2a8b17ee6bbe741ca as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca 2024-11-20T12:15:17,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca, entries=150, sequenceid=308, filesize=30.5 K 2024-11-20T12:15:17,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:17,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/19b7da0135e84a67881bcb9a524c3091 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091 2024-11-20T12:15:17,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:17,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T12:15:17,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb14548b7754ec38ff9963a9f9b94e4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4 2024-11-20T12:15:17,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T12:15:17,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 009722d6d4375a43f57af7eff44b236c in 1270ms, sequenceid=308, compaction requested=true 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:17,790 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:17,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T12:15:17,790 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:17,791 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:17,791 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:17,791 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:17,791 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:17,791 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,791 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,791 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dcc35aadbf34679b03f0ba271bd6729, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.7 K 2024-11-20T12:15:17,791 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=92.2 K 2024-11-20T12:15:17,791 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca] 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcc35aadbf34679b03f0ba271bd6729, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913050 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53c61abf5c9e486cbafc29e70ca80ac4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913050 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e3678a085c14b9c8c1816c45377a90b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732104914209 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5850a2a0cb415f94dc9b22a89fe746, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732104914209 2024-11-20T12:15:17,792 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b7da0135e84a67881bcb9a524c3091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:17,793 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 272c6375255349a2a8b17ee6bbe741ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:17,798 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:17,799 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:17,800 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8237ddc4662146bbb8bf4d65124b8697 is 50, key is test_row_0/B:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:17,802 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120c32d8968da7e4a6f8bfc8ee880bd7fa3_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:17,804 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120c32d8968da7e4a6f8bfc8ee880bd7fa3_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:17,804 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c32d8968da7e4a6f8bfc8ee880bd7fa3_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:17,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742038_1214 (size=13051) 2024-11-20T12:15:17,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742039_1215 (size=4469) 2024-11-20T12:15:17,815 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#177 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:17,816 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/d220ece308354eafa31899b9ad3bf578 is 175, key is test_row_0/A:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:17,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742040_1216 (size=32005) 2024-11-20T12:15:17,827 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/d220ece308354eafa31899b9ad3bf578 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578 2024-11-20T12:15:17,834 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into d220ece308354eafa31899b9ad3bf578(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:17,834 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:17,834 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104917790; duration=0sec 2024-11-20T12:15:17,834 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:17,834 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:17,834 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:17,835 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:17,835 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:17,835 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,835 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/862aad6e33da4a53bfe686cb536be02e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.6 K 2024-11-20T12:15:17,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 862aad6e33da4a53bfe686cb536be02e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732104913050 2024-11-20T12:15:17,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eff2b29df0d4eef8296c5c52d4b535f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732104914209 2024-11-20T12:15:17,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb14548b7754ec38ff9963a9f9b94e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:17,843 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:17,843 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/7b7c72e74740452eb4588f77adf11a23 is 50, key is test_row_0/C:col10/1732104916398/Put/seqid=0 2024-11-20T12:15:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:17,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742041_1217 (size=13017) 2024-11-20T12:15:17,854 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/7b7c72e74740452eb4588f77adf11a23 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/7b7c72e74740452eb4588f77adf11a23 2024-11-20T12:15:17,858 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 7b7c72e74740452eb4588f77adf11a23(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:17,858 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:17,858 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=13, startTime=1732104917790; duration=0sec 2024-11-20T12:15:17,858 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:17,858 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:17,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:17,933 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e1aea67f1c7543feb778a88d3cdfe0c2_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104916524/Put/seqid=0 2024-11-20T12:15:17,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742042_1218 (size=12454) 2024-11-20T12:15:18,213 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/8237ddc4662146bbb8bf4d65124b8697 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8237ddc4662146bbb8bf4d65124b8697 2024-11-20T12:15:18,217 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 8237ddc4662146bbb8bf4d65124b8697(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:18,217 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:18,217 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104917790; duration=0sec 2024-11-20T12:15:18,218 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:18,218 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:18,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:18,354 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e1aea67f1c7543feb778a88d3cdfe0c2_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e1aea67f1c7543feb778a88d3cdfe0c2_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:18,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4400d2b92ded43b6b9b688fee41ae58f, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:18,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4400d2b92ded43b6b9b688fee41ae58f is 175, key is test_row_0/A:col10/1732104916524/Put/seqid=0 2024-11-20T12:15:18,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742043_1219 (size=31255) 2024-11-20T12:15:18,360 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4400d2b92ded43b6b9b688fee41ae58f 2024-11-20T12:15:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b12ea857c69e4590b82147cd6ae54728 is 50, key is test_row_0/B:col10/1732104916524/Put/seqid=0 2024-11-20T12:15:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742044_1220 (size=12301) 2024-11-20T12:15:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:18,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:18,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104978724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104978725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104978726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,788 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b12ea857c69e4590b82147cd6ae54728 2024-11-20T12:15:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/f0325293bc8c45f9944e5af650844dfb is 50, key is test_row_0/C:col10/1732104916524/Put/seqid=0 2024-11-20T12:15:18,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742045_1221 (size=12301) 2024-11-20T12:15:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104978836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104978836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104978837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:18,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:19,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104979046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104979047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104979047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,200 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/f0325293bc8c45f9944e5af650844dfb 2024-11-20T12:15:19,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/4400d2b92ded43b6b9b688fee41ae58f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f 2024-11-20T12:15:19,208 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f, entries=150, sequenceid=324, filesize=30.5 K 2024-11-20T12:15:19,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/b12ea857c69e4590b82147cd6ae54728 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728 2024-11-20T12:15:19,213 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T12:15:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/f0325293bc8c45f9944e5af650844dfb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb 2024-11-20T12:15:19,218 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb, entries=150, sequenceid=324, filesize=12.0 K 2024-11-20T12:15:19,219 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 009722d6d4375a43f57af7eff44b236c in 1286ms, sequenceid=324, compaction requested=false 2024-11-20T12:15:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T12:15:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T12:15:19,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T12:15:19,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5110 sec 2024-11-20T12:15:19,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.5150 sec 2024-11-20T12:15:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:19,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732104918715, max timestamp=1732104919357Num uniques 859; 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732104918715, max timestamp=1732104919357Num uniques 859; 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732104918715, max timestamp=1732104919357Num uniques 859; 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:19,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:19,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209be898e989904316bcf757b21a5fb4b2_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:19,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742046_1222 (size=20074) 2024-11-20T12:15:19,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104919384, max timestamp=1732104919430Num uniques 859; 2024-11-20T12:15:19,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104919384, max timestamp=1732104919430Num uniques 859; 2024-11-20T12:15:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104919384, max timestamp=1732104919430Num uniques 859; 2024-11-20T12:15:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:19,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:19,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1732104919443, max timestamp=1732104919465Num uniques 1068; 2024-11-20T12:15:19,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1732104919443, max timestamp=1732104919465Num uniques 1068; 2024-11-20T12:15:19,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-11-20T12:15:19,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-11-20T12:15:19,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=468, cellSize=21428, totalHeapSize=48912, min timestamp=1732104919443, max timestamp=1732104919465Num uniques 1068; 2024-11-20T12:15:19,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=21428, suffix heap size=139040,new segment heap size=48912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=468 2024-11-20T12:15:19,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104979463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104979504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104979506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104979576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104979616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104979616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:19,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209be898e989904316bcf757b21a5fb4b2_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209be898e989904316bcf757b21a5fb4b2_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:19,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0a165992da94434da40beba8eb9c9c4b, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:19,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0a165992da94434da40beba8eb9c9c4b is 175, key is test_row_0/A:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104979787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742047_1223 (size=57333) 2024-11-20T12:15:19,794 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0a165992da94434da40beba8eb9c9c4b 2024-11-20T12:15:19,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/782ebe6ee9784abfa27c627051f8c75b is 50, key is test_row_0/B:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:19,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742048_1224 (size=12301) 2024-11-20T12:15:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104979826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104979826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104980096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104980137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104980137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/782ebe6ee9784abfa27c627051f8c75b 2024-11-20T12:15:20,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/40d3c23851fa4113bcfd300025c92193 is 50, key is test_row_0/C:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:20,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742049_1225 (size=12301) 2024-11-20T12:15:20,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104980606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/40d3c23851fa4113bcfd300025c92193 2024-11-20T12:15:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104980647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104980647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:20,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/0a165992da94434da40beba8eb9c9c4b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b 2024-11-20T12:15:20,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b, entries=300, sequenceid=346, filesize=56.0 K 2024-11-20T12:15:20,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/782ebe6ee9784abfa27c627051f8c75b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b 2024-11-20T12:15:20,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T12:15:20,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/40d3c23851fa4113bcfd300025c92193 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193 2024-11-20T12:15:20,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193, entries=150, sequenceid=346, filesize=12.0 K 2024-11-20T12:15:20,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~96.66 KB/98976, currentSize=154.31 KB/158010 for 009722d6d4375a43f57af7eff44b236c in 1328ms, sequenceid=346, compaction requested=true 2024-11-20T12:15:20,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:20,686 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:20,686 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:20,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:20,686 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:20,686 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/B is initiating minor compaction (all files) 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/A is initiating minor compaction (all files) 2024-11-20T12:15:20,687 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/B in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:20,687 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/A in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:20,687 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8237ddc4662146bbb8bf4d65124b8697, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.8 K 2024-11-20T12:15:20,687 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=117.8 K 2024-11-20T12:15:20,687 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b] 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8237ddc4662146bbb8bf4d65124b8697, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d220ece308354eafa31899b9ad3bf578, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:20,687 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b12ea857c69e4590b82147cd6ae54728, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732104916524 2024-11-20T12:15:20,688 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4400d2b92ded43b6b9b688fee41ae58f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732104916524 2024-11-20T12:15:20,688 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 782ebe6ee9784abfa27c627051f8c75b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732104918724 2024-11-20T12:15:20,688 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a165992da94434da40beba8eb9c9c4b, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732104918715 2024-11-20T12:15:20,693 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:20,695 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:20,695 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/123cb49de1e442a6b222bd745701cc8e is 50, key is test_row_0/B:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:20,701 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120faeacf97b5d244c2afdc639123a1cb7c_009722d6d4375a43f57af7eff44b236c store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:20,703 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120faeacf97b5d244c2afdc639123a1cb7c_009722d6d4375a43f57af7eff44b236c, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:20,703 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120faeacf97b5d244c2afdc639123a1cb7c_009722d6d4375a43f57af7eff44b236c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:20,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742050_1226 (size=13153) 2024-11-20T12:15:20,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742051_1227 (size=4469) 2024-11-20T12:15:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T12:15:20,877 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:20,877 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-20T12:15:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:20,879 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:20,880 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:20,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:20,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:21,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-20T12:15:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:21,032 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=154.31 KB heapSize=415.78 KB 2024-11-20T12:15:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:21,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:21,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b84f03e4d5548989eae99af01ab0918_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104919505/Put/seqid=0 2024-11-20T12:15:21,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742052_1228 (size=12454) 2024-11-20T12:15:21,108 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#A#compaction#185 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:21,108 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/211917de0c5345cf92457ceeb1fd0452 is 175, key is test_row_0/A:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. as already flushing 2024-11-20T12:15:21,111 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/123cb49de1e442a6b222bd745701cc8e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/123cb49de1e442a6b222bd745701cc8e 2024-11-20T12:15:21,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742053_1229 (size=32107) 2024-11-20T12:15:21,116 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/B of 009722d6d4375a43f57af7eff44b236c into 123cb49de1e442a6b222bd745701cc8e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:21,116 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:21,116 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/B, priority=13, startTime=1732104920686; duration=0sec 2024-11-20T12:15:21,116 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:21,116 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:21,116 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:21,118 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:21,118 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): 009722d6d4375a43f57af7eff44b236c/C is initiating minor compaction (all files) 2024-11-20T12:15:21,118 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 009722d6d4375a43f57af7eff44b236c/C in TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:21,118 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/7b7c72e74740452eb4588f77adf11a23, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp, totalSize=36.7 K 2024-11-20T12:15:21,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104981123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104981124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,130 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7c72e74740452eb4588f77adf11a23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732104916398 2024-11-20T12:15:21,130 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f0325293bc8c45f9944e5af650844dfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732104916524 2024-11-20T12:15:21,130 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d3c23851fa4113bcfd300025c92193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732104918724 2024-11-20T12:15:21,138 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 009722d6d4375a43f57af7eff44b236c#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:21,138 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9b822c10018d402aae14129a0c335b86 is 50, key is test_row_0/C:col10/1732104919357/Put/seqid=0 2024-11-20T12:15:21,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742054_1230 (size=13119) 2024-11-20T12:15:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104981227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104981237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104981436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:21,448 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205b84f03e4d5548989eae99af01ab0918_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b84f03e4d5548989eae99af01ab0918_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:21,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ac315b454c6348e69ae8d2a2b6f9d1a9, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:21,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ac315b454c6348e69ae8d2a2b6f9d1a9 is 175, key is test_row_0/A:col10/1732104919505/Put/seqid=0 2024-11-20T12:15:21,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104981447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742055_1231 (size=31255) 2024-11-20T12:15:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,455 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ac315b454c6348e69ae8d2a2b6f9d1a9 2024-11-20T12:15:21,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/f5de170139f6497ba107caa17a5743aa is 50, key is test_row_0/B:col10/1732104919505/Put/seqid=0 2024-11-20T12:15:21,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742056_1232 (size=12301) 2024-11-20T12:15:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:21,517 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/211917de0c5345cf92457ceeb1fd0452 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/211917de0c5345cf92457ceeb1fd0452 2024-11-20T12:15:21,522 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/A of 009722d6d4375a43f57af7eff44b236c into 211917de0c5345cf92457ceeb1fd0452(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:21,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:21,522 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/A, priority=13, startTime=1732104920685; duration=0sec 2024-11-20T12:15:21,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:21,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:21,546 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/9b822c10018d402aae14129a0c335b86 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b822c10018d402aae14129a0c335b86 2024-11-20T12:15:21,551 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 009722d6d4375a43f57af7eff44b236c/C of 009722d6d4375a43f57af7eff44b236c into 9b822c10018d402aae14129a0c335b86(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:21,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:21,551 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., storeName=009722d6d4375a43f57af7eff44b236c/C, priority=13, startTime=1732104920686; duration=0sec 2024-11-20T12:15:21,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:21,551 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56374 deadline: 1732104981616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,621 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56362 deadline: 1732104981657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56354 deadline: 1732104981657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,659 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,659 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in 009722d6d4375a43f57af7eff44b236c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,710 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-20T12:15:21,710 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:21,710 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,711 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,711 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:21,714 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-20T12:15:21,714 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-20T12:15:21,714 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:21,714 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:21,714 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,714 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,714 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,714 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:21,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:21,716 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-20T12:15:21,716 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:21,716 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,716 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:21,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:21,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104981747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104981757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:21,869 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/f5de170139f6497ba107caa17a5743aa 2024-11-20T12:15:21,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb2d56c5ccc46f0924567d5f15fbed5 is 50, key is test_row_0/C:col10/1732104919505/Put/seqid=0 2024-11-20T12:15:21,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742057_1233 (size=12301) 2024-11-20T12:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:22,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56388 deadline: 1732104982258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:22,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56342 deadline: 1732104982267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:22,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:22,291 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb2d56c5ccc46f0924567d5f15fbed5 2024-11-20T12:15:22,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/ac315b454c6348e69ae8d2a2b6f9d1a9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ac315b454c6348e69ae8d2a2b6f9d1a9 2024-11-20T12:15:22,309 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ac315b454c6348e69ae8d2a2b6f9d1a9, entries=150, sequenceid=397, filesize=30.5 K 2024-11-20T12:15:22,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/f5de170139f6497ba107caa17a5743aa as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/f5de170139f6497ba107caa17a5743aa 2024-11-20T12:15:22,315 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/f5de170139f6497ba107caa17a5743aa, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T12:15:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/efb2d56c5ccc46f0924567d5f15fbed5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb2d56c5ccc46f0924567d5f15fbed5 2024-11-20T12:15:22,320 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb2d56c5ccc46f0924567d5f15fbed5, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T12:15:22,321 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~415.73 KB/425712, currentSize=33.54 KB/34350 for 009722d6d4375a43f57af7eff44b236c in 1289ms, sequenceid=397, compaction requested=false 2024-11-20T12:15:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-20T12:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-20T12:15:22,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T12:15:22,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4430 sec 2024-11-20T12:15:22,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4470 sec 2024-11-20T12:15:22,704 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:15:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T12:15:23,037 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:23,271 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-20T12:15:23,271 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:23,271 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,271 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:23,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:23,289 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-20T12:15:23,289 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:23,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:23,289 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,289 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:23,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:23,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:23,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:23,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201545e389be0047989125c61985b09399_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104923287/Put/seqid=0 2024-11-20T12:15:23,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742058_1234 (size=12454) 2024-11-20T12:15:23,641 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-20T12:15:23,641 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:23,641 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,642 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,671 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-20T12:15:23,671 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:23,671 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,672 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,679 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-20T12:15:23,680 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:23,680 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,680 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,680 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3370 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3088 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:15:23,680 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1358 2024-11-20T12:15:23,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4073 rows 2024-11-20T12:15:23,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1362 2024-11-20T12:15:23,681 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4080 rows 2024-11-20T12:15:23,681 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:15:23,681 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:15:23,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:23,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:23,683 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:15:23,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:23,688 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104923688"}]},"ts":"1732104923688"} 2024-11-20T12:15:23,690 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:15:23,690 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:15:23,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:15:23,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, UNASSIGN}] 2024-11-20T12:15:23,694 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, UNASSIGN 2024-11-20T12:15:23,695 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:23,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, UNASSIGN because future has completed 2024-11-20T12:15:23,697 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:15:23,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:23,704 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:23,708 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201545e389be0047989125c61985b09399_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201545e389be0047989125c61985b09399_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:23,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/72ac5412882b4c0c89268a52b527f3f1, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:23,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/72ac5412882b4c0c89268a52b527f3f1 is 175, key is test_row_0/A:col10/1732104923287/Put/seqid=0 2024-11-20T12:15:23,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742059_1235 (size=31255) 2024-11-20T12:15:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:23,851 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:23,851 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:15:23,852 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 009722d6d4375a43f57af7eff44b236c, disabling compactions & flushes 2024-11-20T12:15:23,852 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:24,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:24,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/72ac5412882b4c0c89268a52b527f3f1 2024-11-20T12:15:24,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6501f5a44187453581a9fbdfa3dde8b3 is 50, key is test_row_0/B:col10/1732104923287/Put/seqid=0 2024-11-20T12:15:24,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742060_1236 (size=12301) 2024-11-20T12:15:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:24,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6501f5a44187453581a9fbdfa3dde8b3 2024-11-20T12:15:24,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/2f7af6ecba00482a8281b5317855ae4f is 50, key is test_row_0/C:col10/1732104923287/Put/seqid=0 2024-11-20T12:15:24,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742061_1237 (size=12301) 2024-11-20T12:15:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:24,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/2f7af6ecba00482a8281b5317855ae4f 2024-11-20T12:15:24,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/72ac5412882b4c0c89268a52b527f3f1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/72ac5412882b4c0c89268a52b527f3f1 2024-11-20T12:15:24,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/72ac5412882b4c0c89268a52b527f3f1, entries=150, sequenceid=410, filesize=30.5 K 2024-11-20T12:15:24,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/6501f5a44187453581a9fbdfa3dde8b3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6501f5a44187453581a9fbdfa3dde8b3 2024-11-20T12:15:24,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6501f5a44187453581a9fbdfa3dde8b3, entries=150, sequenceid=410, filesize=12.0 K 2024-11-20T12:15:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/2f7af6ecba00482a8281b5317855ae4f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/2f7af6ecba00482a8281b5317855ae4f 2024-11-20T12:15:24,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/2f7af6ecba00482a8281b5317855ae4f, entries=150, sequenceid=410, filesize=12.0 K 2024-11-20T12:15:24,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 009722d6d4375a43f57af7eff44b236c in 1694ms, sequenceid=410, compaction requested=true 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 009722d6d4375a43f57af7eff44b236c: 2024-11-20T12:15:24,982 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:24,982 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. because compaction request was cancelled 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:A 2024-11-20T12:15:24,982 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. after waiting 0 ms 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:24,982 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. because compaction request was cancelled 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:B 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 009722d6d4375a43f57af7eff44b236c:C, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. because compaction request was cancelled 2024-11-20T12:15:24,982 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 009722d6d4375a43f57af7eff44b236c:C 2024-11-20T12:15:24,982 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 009722d6d4375a43f57af7eff44b236c 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=A 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=B 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 009722d6d4375a43f57af7eff44b236c, store=C 2024-11-20T12:15:24,983 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:24,988 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039fda9b3b24f45f98c7997f22d30c02c_009722d6d4375a43f57af7eff44b236c is 50, key is test_row_0/A:col10/1732104923678/Put/seqid=0 2024-11-20T12:15:24,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742062_1238 (size=9914) 2024-11-20T12:15:25,393 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:25,403 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112039fda9b3b24f45f98c7997f22d30c02c_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039fda9b3b24f45f98c7997f22d30c02c_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:25,404 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/f1f501a7d3ef4871a68fe670b38b4892, store: [table=TestAcidGuarantees family=A region=009722d6d4375a43f57af7eff44b236c] 2024-11-20T12:15:25,405 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/f1f501a7d3ef4871a68fe670b38b4892 is 175, key is test_row_0/A:col10/1732104923678/Put/seqid=0 2024-11-20T12:15:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742063_1239 (size=22561) 2024-11-20T12:15:25,810 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/f1f501a7d3ef4871a68fe670b38b4892 2024-11-20T12:15:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:25,881 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4173714f77a94a0db08e51f7c30d65df is 50, key is test_row_0/B:col10/1732104923678/Put/seqid=0 2024-11-20T12:15:25,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742064_1240 (size=9857) 2024-11-20T12:15:26,286 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4173714f77a94a0db08e51f7c30d65df 2024-11-20T12:15:26,297 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/a7db0292e78a463380b842c0c9b61880 is 50, key is test_row_0/C:col10/1732104923678/Put/seqid=0 2024-11-20T12:15:26,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742065_1241 (size=9857) 2024-11-20T12:15:26,703 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/a7db0292e78a463380b842c0c9b61880 2024-11-20T12:15:26,711 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/A/f1f501a7d3ef4871a68fe670b38b4892 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/f1f501a7d3ef4871a68fe670b38b4892 2024-11-20T12:15:26,716 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/f1f501a7d3ef4871a68fe670b38b4892, entries=100, sequenceid=416, filesize=22.0 K 2024-11-20T12:15:26,717 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/B/4173714f77a94a0db08e51f7c30d65df as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4173714f77a94a0db08e51f7c30d65df 2024-11-20T12:15:26,722 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4173714f77a94a0db08e51f7c30d65df, entries=100, sequenceid=416, filesize=9.6 K 2024-11-20T12:15:26,723 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/.tmp/C/a7db0292e78a463380b842c0c9b61880 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/a7db0292e78a463380b842c0c9b61880 2024-11-20T12:15:26,726 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/a7db0292e78a463380b842c0c9b61880, entries=100, sequenceid=416, filesize=9.6 K 2024-11-20T12:15:26,727 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 009722d6d4375a43f57af7eff44b236c in 1745ms, sequenceid=416, compaction requested=true 2024-11-20T12:15:26,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b] to archive 2024-11-20T12:15:26,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:26,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/06a3bd69263b4376b61a5ec6a01668c3 2024-11-20T12:15:26,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1e66ac4310a744ed9e11dcf0339425a3 2024-11-20T12:15:26,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/5ec8a96e7f7040639561c2520a88e2e3 2024-11-20T12:15:26,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/c94d5c0a708046e09db63445b0ff5a59 2024-11-20T12:15:26,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3bb231aa9c2d444a88c7159ce2d0f765 2024-11-20T12:15:26,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/671a1b2a47ff43499c377bd47c954b0b 2024-11-20T12:15:26,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/baf2f924bf904a8aadad418d1af09306 2024-11-20T12:15:26,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/eeb1c6cde1e54f139f6a522ccc3fd391 2024-11-20T12:15:26,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ab9b3850f88a47b8b53fbfd7f047dacf 2024-11-20T12:15:26,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d5ee60a4d9df421db216a7e21eed4194 2024-11-20T12:15:26,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ae7aeef66f84a289d6237f5c8e037e2 2024-11-20T12:15:26,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85b43ab7b462408a9300dc66bac058d5 2024-11-20T12:15:26,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/85e2619803aa459ebed97b3e54a0e978 2024-11-20T12:15:26,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/26a3f68e25c64e31af31df8c1714bb65 2024-11-20T12:15:26,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0b4bceb5e7d24ef7b1a35328ca7b1f30 2024-11-20T12:15:26,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/80b5feabe3794978832c65d655578b2c 2024-11-20T12:15:26,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4ed367f49dff4d5ebce0e980d888a33c 2024-11-20T12:15:26,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/1093d551e4a84805973677ed97e42938 2024-11-20T12:15:26,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4602c7163e304f35befeba79b01423f8 2024-11-20T12:15:26,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/53c61abf5c9e486cbafc29e70ca80ac4 2024-11-20T12:15:26,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/3a5850a2a0cb415f94dc9b22a89fe746 2024-11-20T12:15:26,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/d220ece308354eafa31899b9ad3bf578 2024-11-20T12:15:26,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/272c6375255349a2a8b17ee6bbe741ca 2024-11-20T12:15:26,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/4400d2b92ded43b6b9b688fee41ae58f 2024-11-20T12:15:26,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/0a165992da94434da40beba8eb9c9c4b 2024-11-20T12:15:26,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/40abecb1980d40559e915eb58a6030e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/94ae7778c73544159d9b3ee1071f0615, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/037344539f3145a08448b77cc2b5cd15, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8bc492c71a6243af8dec19d8311f9e6f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/cc509e282442442195fd6c24677443b4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dcc35aadbf34679b03f0ba271bd6729, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8237ddc4662146bbb8bf4d65124b8697, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b] to archive 2024-11-20T12:15:26,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:26,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b336c19735c74f309df8e4a286b2ddd7 2024-11-20T12:15:26,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dc812944a69442884146fa5b81509e4 2024-11-20T12:15:26,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/40abecb1980d40559e915eb58a6030e0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/40abecb1980d40559e915eb58a6030e0 2024-11-20T12:15:26,760 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d4a457b3e5884d98a2a72c0bca973eac 2024-11-20T12:15:26,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/c1d1c2c6561b47c2ab931d7ea87a0f40 2024-11-20T12:15:26,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/94ae7778c73544159d9b3ee1071f0615 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/94ae7778c73544159d9b3ee1071f0615 2024-11-20T12:15:26,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/938bbb9e23524225b092262696ea6b33 2024-11-20T12:15:26,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6357260e361e437d9fa342eea0e88843 2024-11-20T12:15:26,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/037344539f3145a08448b77cc2b5cd15 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/037344539f3145a08448b77cc2b5cd15 2024-11-20T12:15:26,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/769179bdfed7472d8c1d0f83debbe4d8 2024-11-20T12:15:26,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/ba8f4dbca8e94307a1443e682c252e3c 2024-11-20T12:15:26,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8bc492c71a6243af8dec19d8311f9e6f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8bc492c71a6243af8dec19d8311f9e6f 2024-11-20T12:15:26,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/aa9f5091189a4c3c89385f18eec6a844 2024-11-20T12:15:26,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/d7d0295d1a4e405191d8feeaf402e4ab 2024-11-20T12:15:26,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/cc509e282442442195fd6c24677443b4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/cc509e282442442195fd6c24677443b4 2024-11-20T12:15:26,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/612a2a9aa7664451ab464107a04cbb00 2024-11-20T12:15:26,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4dbe3f0381af428a868bff7a001e27ab 2024-11-20T12:15:26,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8c736103d7ae4689a7b657af58ddffca 2024-11-20T12:15:26,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dcc35aadbf34679b03f0ba271bd6729 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/9dcc35aadbf34679b03f0ba271bd6729 2024-11-20T12:15:26,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8f8e99aae6e34b918fe6d23f374c9dbf 2024-11-20T12:15:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/3e3678a085c14b9c8c1816c45377a90b 2024-11-20T12:15:26,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8237ddc4662146bbb8bf4d65124b8697 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/8237ddc4662146bbb8bf4d65124b8697 2024-11-20T12:15:26,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/19b7da0135e84a67881bcb9a524c3091 2024-11-20T12:15:26,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/b12ea857c69e4590b82147cd6ae54728 2024-11-20T12:15:26,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/782ebe6ee9784abfa27c627051f8c75b 2024-11-20T12:15:26,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/71dbe860d791408d8301e4859bf2ca16, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ae81a35d7b5a4b1b935a2b156688be0e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/3731dbe8013445ef818d06170f841f01, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/8e595f6efea747d9852cc36f400f1750, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/862aad6e33da4a53bfe686cb536be02e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/7b7c72e74740452eb4588f77adf11a23, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193] to archive 2024-11-20T12:15:26,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:26,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/21e1b508ae5f437f87875860b7022c4a 2024-11-20T12:15:26,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0fc16fc1445847bf8e9f76af94d33355 2024-11-20T12:15:26,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9f57045dfbfd4204bd0e3beade233d3c 2024-11-20T12:15:26,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/71dbe860d791408d8301e4859bf2ca16 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/71dbe860d791408d8301e4859bf2ca16 2024-11-20T12:15:26,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b17fac427b14d85be41041af494c65b 2024-11-20T12:15:26,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ff89aac1389f40ccabe07b5fdfb28954 2024-11-20T12:15:26,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/defb2110686248a886118809c893fdf5 2024-11-20T12:15:26,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ae81a35d7b5a4b1b935a2b156688be0e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/ae81a35d7b5a4b1b935a2b156688be0e 2024-11-20T12:15:26,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/91b6e977a890414599deb8b905691e4e 2024-11-20T12:15:26,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/c8e761f275c845b7a9e14010f791d8a3 2024-11-20T12:15:26,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/3731dbe8013445ef818d06170f841f01 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/3731dbe8013445ef818d06170f841f01 2024-11-20T12:15:26,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/048fc888e7164a25b726c02ea6613090 2024-11-20T12:15:26,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6672f56186f949eda949bb94566ddf8c 2024-11-20T12:15:26,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/8e595f6efea747d9852cc36f400f1750 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/8e595f6efea747d9852cc36f400f1750 2024-11-20T12:15:26,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/d37ea0c3d2c54e47a2227afdcc994e7d 2024-11-20T12:15:26,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/0f5f893e684f43c997e2863ae7a45355 2024-11-20T12:15:26,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/fe7d6bdb64b8448b85bf202f8af8ba4b 2024-11-20T12:15:26,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/862aad6e33da4a53bfe686cb536be02e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/862aad6e33da4a53bfe686cb536be02e 2024-11-20T12:15:26,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/afe59aa6b8fd4dd2ab22f03b932d568c 2024-11-20T12:15:26,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/6eff2b29df0d4eef8296c5c52d4b535f 2024-11-20T12:15:26,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/7b7c72e74740452eb4588f77adf11a23 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/7b7c72e74740452eb4588f77adf11a23 2024-11-20T12:15:26,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb14548b7754ec38ff9963a9f9b94e4 2024-11-20T12:15:26,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/f0325293bc8c45f9944e5af650844dfb 2024-11-20T12:15:26,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/40d3c23851fa4113bcfd300025c92193 2024-11-20T12:15:26,804 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits/419.seqid, newMaxSeqId=419, maxSeqId=4 2024-11-20T12:15:26,805 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. 2024-11-20T12:15:26,805 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 009722d6d4375a43f57af7eff44b236c: Waiting for close lock at 1732104923852Running coprocessor pre-close hooks at 1732104923852Disabling compacts and flushes for region at 1732104923852Disabling writes for close at 1732104924982 (+1130 ms)Obtaining lock to block concurrent updates at 1732104924982Preparing flush snapshotting stores in 009722d6d4375a43f57af7eff44b236c at 1732104924982Finished memstore snapshotting TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732104924983 (+1 ms)Flushing stores of TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c. at 1732104924983Flushing 009722d6d4375a43f57af7eff44b236c/A: creating writer at 1732104924983Flushing mob file 009722d6d4375a43f57af7eff44b236c/A: appending metadata at 1732104924988 (+5 ms)Flushing mob file 009722d6d4375a43f57af7eff44b236c/A: closing flushed file at 1732104924988Flushing 009722d6d4375a43f57af7eff44b236c/A: appending metadata at 1732104925404 (+416 ms)Flushing 009722d6d4375a43f57af7eff44b236c/A: closing flushed file at 1732104925404Flushing 009722d6d4375a43f57af7eff44b236c/B: creating writer at 1732104925874 (+470 ms)Flushing 009722d6d4375a43f57af7eff44b236c/B: appending metadata at 1732104925880 (+6 ms)Flushing 009722d6d4375a43f57af7eff44b236c/B: closing flushed file at 1732104925881 (+1 ms)Flushing 009722d6d4375a43f57af7eff44b236c/C: creating writer at 1732104926294 (+413 ms)Flushing 009722d6d4375a43f57af7eff44b236c/C: appending metadata at 1732104926297 (+3 ms)Flushing 009722d6d4375a43f57af7eff44b236c/C: closing flushed file at 1732104926297Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5903c1df: reopening flushed file at 1732104926710 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ca9a892: reopening flushed file at 1732104926716 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3239dba2: reopening flushed file at 1732104926722 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 009722d6d4375a43f57af7eff44b236c in 1745ms, sequenceid=416, compaction requested=true at 1732104926727 (+5 ms)Writing region close event to WAL at 1732104926801 (+74 ms)Running coprocessor post-close hooks at 1732104926804 (+3 ms)Closed at 1732104926805 (+1 ms) 2024-11-20T12:15:26,807 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:26,807 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=009722d6d4375a43f57af7eff44b236c, regionState=CLOSED 2024-11-20T12:15:26,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:26,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-20T12:15:26,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 009722d6d4375a43f57af7eff44b236c, server=843d83d51ba0,42825,1732104864746 in 3.1120 sec 2024-11-20T12:15:26,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T12:15:26,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=009722d6d4375a43f57af7eff44b236c, UNASSIGN in 3.1180 sec 2024-11-20T12:15:26,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T12:15:26,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.1220 sec 2024-11-20T12:15:26,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104926815"}]},"ts":"1732104926815"} 2024-11-20T12:15:26,817 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:15:26,817 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:15:26,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.1340 sec 2024-11-20T12:15:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-20T12:15:27,848 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:27,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:15:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,855 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T12:15:27,856 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,860 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,865 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits] 2024-11-20T12:15:27,870 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/211917de0c5345cf92457ceeb1fd0452 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/211917de0c5345cf92457ceeb1fd0452 2024-11-20T12:15:27,872 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/72ac5412882b4c0c89268a52b527f3f1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/72ac5412882b4c0c89268a52b527f3f1 2024-11-20T12:15:27,874 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ac315b454c6348e69ae8d2a2b6f9d1a9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/ac315b454c6348e69ae8d2a2b6f9d1a9 2024-11-20T12:15:27,876 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/f1f501a7d3ef4871a68fe670b38b4892 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/A/f1f501a7d3ef4871a68fe670b38b4892 2024-11-20T12:15:27,880 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/123cb49de1e442a6b222bd745701cc8e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/123cb49de1e442a6b222bd745701cc8e 2024-11-20T12:15:27,881 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4173714f77a94a0db08e51f7c30d65df to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/4173714f77a94a0db08e51f7c30d65df 2024-11-20T12:15:27,882 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6501f5a44187453581a9fbdfa3dde8b3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/6501f5a44187453581a9fbdfa3dde8b3 2024-11-20T12:15:27,884 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/f5de170139f6497ba107caa17a5743aa to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/B/f5de170139f6497ba107caa17a5743aa 2024-11-20T12:15:27,886 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/2f7af6ecba00482a8281b5317855ae4f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/2f7af6ecba00482a8281b5317855ae4f 2024-11-20T12:15:27,887 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b822c10018d402aae14129a0c335b86 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/9b822c10018d402aae14129a0c335b86 2024-11-20T12:15:27,887 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/a7db0292e78a463380b842c0c9b61880 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/a7db0292e78a463380b842c0c9b61880 2024-11-20T12:15:27,888 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb2d56c5ccc46f0924567d5f15fbed5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/C/efb2d56c5ccc46f0924567d5f15fbed5 2024-11-20T12:15:27,890 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits/419.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c/recovered.edits/419.seqid 2024-11-20T12:15:27,891 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,891 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:15:27,892 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:15:27,893 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T12:15:27,896 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120086cfa6a1c184d66a622d3ec318a8684_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120086cfa6a1c184d66a622d3ec318a8684_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,898 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c7cec672d554a64a14397ab5335a4e3_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200c7cec672d554a64a14397ab5335a4e3_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,899 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012086e8719264a14991da29383036249_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112012086e8719264a14991da29383036249_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,900 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201545e389be0047989125c61985b09399_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201545e389be0047989125c61985b09399_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,901 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016df4aa6e85c422ebac030774c594b42_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016df4aa6e85c422ebac030774c594b42_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,902 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a0eff66b6e443bb908be936f6bece75_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a0eff66b6e443bb908be936f6bece75_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,903 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039fda9b3b24f45f98c7997f22d30c02c_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112039fda9b3b24f45f98c7997f22d30c02c_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,904 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ad69d861d5341559b93ab76a5c46ba6_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203ad69d861d5341559b93ab76a5c46ba6_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,905 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203debf88fd4134d369329a75945c3920e_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203debf88fd4134d369329a75945c3920e_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,906 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b84f03e4d5548989eae99af01ab0918_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205b84f03e4d5548989eae99af01ab0918_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,907 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112071b42f0af2c94387a7a9f697b1772578_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112071b42f0af2c94387a7a9f697b1772578_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,908 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073e0a9759e204e3aa1c8703370130d68_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112073e0a9759e204e3aa1c8703370130d68_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,908 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207721baa6e98d4daeafc28f84f6c941c6_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207721baa6e98d4daeafc28f84f6c941c6_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,909 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209be898e989904316bcf757b21a5fb4b2_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209be898e989904316bcf757b21a5fb4b2_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,910 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a118e9b2832a4c1aafd3cb8a0579c66d_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a118e9b2832a4c1aafd3cb8a0579c66d_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,911 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af04a223965d48fa9364271650573d0b_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120af04a223965d48fa9364271650573d0b_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,912 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31800693fc04c9d95417aed09db0bad_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b31800693fc04c9d95417aed09db0bad_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,912 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd47662fc99b43f295d0bbffa1fa1133_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bd47662fc99b43f295d0bbffa1fa1133_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,913 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d41526e1c2d54766ad9062d71fdf86f8_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d41526e1c2d54766ad9062d71fdf86f8_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,914 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e1aea67f1c7543feb778a88d3cdfe0c2_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e1aea67f1c7543feb778a88d3cdfe0c2_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,915 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0377a04ef95448397af7d18a954fb13_009722d6d4375a43f57af7eff44b236c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f0377a04ef95448397af7d18a954fb13_009722d6d4375a43f57af7eff44b236c 2024-11-20T12:15:27,915 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:15:27,917 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,919 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:15:27,920 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:15:27,922 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,922 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:15:27,922 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732104927922"}]},"ts":"9223372036854775807"} 2024-11-20T12:15:27,924 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:15:27,924 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 009722d6d4375a43f57af7eff44b236c, NAME => 'TestAcidGuarantees,,1732104898462.009722d6d4375a43f57af7eff44b236c.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:15:27,924 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:15:27,924 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732104927924"}]},"ts":"9223372036854775807"} 2024-11-20T12:15:27,926 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:15:27,927 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 77 msec 2024-11-20T12:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-20T12:15:27,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:15:27,968 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:27,981 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250 (was 247) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1709011412_22 at /127.0.0.1:43044 [Waiting for operation #279] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2022882310_22 at /127.0.0.1:47058 [Waiting for operation #292] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=465 (was 462) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2668 (was 2750) 2024-11-20T12:15:27,992 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=465, ProcessCount=11, AvailableMemoryMB=2668 2024-11-20T12:15:27,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:15:27,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:15:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:27,997 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:15:27,997 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:27,997 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-20T12:15:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:27,998 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:15:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742066_1242 (size=1061) 2024-11-20T12:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:28,409 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:15:28,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742067_1243 (size=53) 2024-11-20T12:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ceaa0a251a1510d6c7c95683bd6968dd, disabling compactions & flushes 2024-11-20T12:15:28,818 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. after waiting 0 ms 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:28,818 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:28,818 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ceaa0a251a1510d6c7c95683bd6968dd: Waiting for close lock at 1732104928818Disabling compacts and flushes for region at 1732104928818Disabling writes for close at 1732104928818Writing region close event to WAL at 1732104928818Closed at 1732104928818 2024-11-20T12:15:28,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:15:28,821 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732104928820"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104928820"}]},"ts":"1732104928820"} 2024-11-20T12:15:28,823 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:15:28,825 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:15:28,825 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104928825"}]},"ts":"1732104928825"} 2024-11-20T12:15:28,827 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:15:28,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, ASSIGN}] 2024-11-20T12:15:28,830 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, ASSIGN 2024-11-20T12:15:28,831 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:15:28,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=ceaa0a251a1510d6c7c95683bd6968dd, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:28,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, ASSIGN because future has completed 2024-11-20T12:15:28,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:29,137 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:29,137 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:15:29,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:29,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,138 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,139 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,141 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:29,141 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceaa0a251a1510d6c7c95683bd6968dd columnFamilyName A 2024-11-20T12:15:29,141 DEBUG [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:29,141 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(327): Store=ceaa0a251a1510d6c7c95683bd6968dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:29,142 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,142 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:29,143 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceaa0a251a1510d6c7c95683bd6968dd columnFamilyName B 2024-11-20T12:15:29,143 DEBUG [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:29,143 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(327): Store=ceaa0a251a1510d6c7c95683bd6968dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:29,143 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,144 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:29,144 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ceaa0a251a1510d6c7c95683bd6968dd columnFamilyName C 2024-11-20T12:15:29,144 DEBUG [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:29,145 INFO [StoreOpener-ceaa0a251a1510d6c7c95683bd6968dd-1 {}] regionserver.HStore(327): Store=ceaa0a251a1510d6c7c95683bd6968dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:29,145 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:29,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,146 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,147 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,147 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,147 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:15:29,148 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,150 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:15:29,151 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened ceaa0a251a1510d6c7c95683bd6968dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65434548, jitterRate=-0.024949252605438232}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:15:29,151 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:29,152 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for ceaa0a251a1510d6c7c95683bd6968dd: Running coprocessor pre-open hook at 1732104929138Writing region info on filesystem at 1732104929138Initializing all the Stores at 1732104929139 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104929139Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104929139Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104929139Cleaning up temporary data from old regions at 1732104929147 (+8 ms)Running coprocessor post-open hooks at 1732104929151 (+4 ms)Region opened successfully at 1732104929152 (+1 ms) 2024-11-20T12:15:29,152 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., pid=63, masterSystemTime=1732104929134 2024-11-20T12:15:29,154 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:29,154 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:29,154 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=ceaa0a251a1510d6c7c95683bd6968dd, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:29,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:29,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T12:15:29,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 in 173 msec 2024-11-20T12:15:29,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-20T12:15:29,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, ASSIGN in 329 msec 2024-11-20T12:15:29,160 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:15:29,160 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104929160"}]},"ts":"1732104929160"} 2024-11-20T12:15:29,161 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:15:29,162 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:15:29,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-20T12:15:30,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-20T12:15:30,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:30,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23cf6174, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,149 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b987be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c187f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,152 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,155 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:15:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:15:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6bed7e79 2024-11-20T12:15:30,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:15:30,157 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:15:30,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6f1434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,159 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40affddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,161 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,161 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140b17b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f1deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,163 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360de83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,164 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef67955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec2896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7745918a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,167 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45274, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9ce9fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,169 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5155197b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cac4aab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6477108e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0e849b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,174 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e2b41a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2104318a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6935c50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,178 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af97d9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,179 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a7b367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767db965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,185 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3886096d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c70ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6493a71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,189 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688037bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,190 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6efcaeeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,192 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@416ec45c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,192 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,194 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346609c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727224d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:30,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:30,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764c0fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:30,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:30,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:30,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15768cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:30,200 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,200 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,200 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,200 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,200 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,204 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,205 DEBUG [Thread-1144 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,208 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,210 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:30,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:30,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:30,226 DEBUG [Thread-1148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,228 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,230 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:30,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:30,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:30,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:15:30,245 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:30,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-20T12:15:30,248 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T12:15:30,249 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:30,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:30,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104930261, max timestamp=1732104930275Num uniques 859; 2024-11-20T12:15:30,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104930261, max timestamp=1732104930275Num uniques 859; 2024-11-20T12:15:30,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:30,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:30,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104930261, max timestamp=1732104930275Num uniques 859; 2024-11-20T12:15:30,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:30,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f97598a071484b65a6d5d29fb3a7b4d4 is 50, key is test_row_0/A:col10/1732104930218/Put/seqid=0 2024-11-20T12:15:30,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742068_1244 (size=12001) 2024-11-20T12:15:30,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f97598a071484b65a6d5d29fb3a7b4d4 2024-11-20T12:15:30,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104990316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104990316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104990318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104990327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104990327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/223b131a87a44f93b2233022b34efc0c is 50, key is test_row_0/B:col10/1732104930218/Put/seqid=0 2024-11-20T12:15:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T12:15:30,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742069_1245 (size=12001) 2024-11-20T12:15:30,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/223b131a87a44f93b2233022b34efc0c 2024-11-20T12:15:30,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T12:15:30,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:30,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:30,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:30,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:30,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:30,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/00772f4b731b48058b5cf1e1043ea150 is 50, key is test_row_0/C:col10/1732104930218/Put/seqid=0 2024-11-20T12:15:30,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742070_1246 (size=12001) 2024-11-20T12:15:30,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/00772f4b731b48058b5cf1e1043ea150 2024-11-20T12:15:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104990437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104990437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f97598a071484b65a6d5d29fb3a7b4d4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4 2024-11-20T12:15:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104990437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T12:15:30,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/223b131a87a44f93b2233022b34efc0c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c 2024-11-20T12:15:30,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104990447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104990447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T12:15:30,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/00772f4b731b48058b5cf1e1043ea150 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150 2024-11-20T12:15:30,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150, entries=150, sequenceid=12, filesize=11.7 K 2024-11-20T12:15:30,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ceaa0a251a1510d6c7c95683bd6968dd in 241ms, sequenceid=12, compaction requested=false 2024-11-20T12:15:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T12:15:30,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:30,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T12:15:30,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:30,557 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:30,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:30,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c74e4aa9f6484e8abb72ec68ca540532 is 50, key is test_row_0/A:col10/1732104930325/Put/seqid=0 2024-11-20T12:15:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T12:15:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742071_1247 (size=12001) 2024-11-20T12:15:30,584 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c74e4aa9f6484e8abb72ec68ca540532 2024-11-20T12:15:30,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e32c471a99234a80a21a4e9ab562a33e is 50, key is test_row_0/B:col10/1732104930325/Put/seqid=0 2024-11-20T12:15:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:30,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742072_1248 (size=12001) 2024-11-20T12:15:30,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104990662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104990661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104990698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104990698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104990698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104990808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104990808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104990808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104990817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104990817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T12:15:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104991017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104991017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104991017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104991027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104991027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,067 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e32c471a99234a80a21a4e9ab562a33e 2024-11-20T12:15:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/86d98c035e9249c593c29627abe00fc0 is 50, key is test_row_0/C:col10/1732104930325/Put/seqid=0 2024-11-20T12:15:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742073_1249 (size=12001) 2024-11-20T12:15:31,148 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/86d98c035e9249c593c29627abe00fc0 2024-11-20T12:15:31,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c74e4aa9f6484e8abb72ec68ca540532 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532 2024-11-20T12:15:31,162 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:15:31,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e32c471a99234a80a21a4e9ab562a33e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e 2024-11-20T12:15:31,173 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:15:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/86d98c035e9249c593c29627abe00fc0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0 2024-11-20T12:15:31,191 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:15:31,193 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for ceaa0a251a1510d6c7c95683bd6968dd in 637ms, sequenceid=49, compaction requested=false 2024-11-20T12:15:31,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:31,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-20T12:15:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-20T12:15:31,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T12:15:31,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 946 msec 2024-11-20T12:15:31,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 963 msec 2024-11-20T12:15:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:31,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:15:31,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:31,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:31,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:31,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:31,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:31,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:31,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7d352e5be124fa6b1953427031e5d20 is 50, key is test_row_0/A:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:31,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742074_1250 (size=12001) 2024-11-20T12:15:31,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104991363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104991363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104991367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104991364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104991367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T12:15:31,387 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:31,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T12:15:31,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:31,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:31,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:31,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104991477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104991477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104991477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104991478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104991478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:31,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:31,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104991687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104991687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104991687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104991687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104991687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:31,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:31,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:31,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7d352e5be124fa6b1953427031e5d20 2024-11-20T12:15:31,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/88771729cbd74122ada8edeadd0505f6 is 50, key is test_row_0/B:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:31,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742075_1251 (size=12001) 2024-11-20T12:15:31,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/88771729cbd74122ada8edeadd0505f6 2024-11-20T12:15:31,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/ade35d9f0de6433b9408c238c3e53b6c is 50, key is test_row_0/C:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:31,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:31,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742076_1252 (size=12001) 2024-11-20T12:15:31,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/ade35d9f0de6433b9408c238c3e53b6c 2024-11-20T12:15:31,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7d352e5be124fa6b1953427031e5d20 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20 2024-11-20T12:15:31,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T12:15:31,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/88771729cbd74122ada8edeadd0505f6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6 2024-11-20T12:15:31,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T12:15:31,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/ade35d9f0de6433b9408c238c3e53b6c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c 2024-11-20T12:15:31,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c, entries=150, sequenceid=62, filesize=11.7 K 2024-11-20T12:15:31,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ceaa0a251a1510d6c7c95683bd6968dd in 575ms, sequenceid=62, compaction requested=true 2024-11-20T12:15:31,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:31,908 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:31,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:31,908 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:31,909 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:31,909 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:31,909 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:31,909 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:31,909 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,909 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,909 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.2 K 2024-11-20T12:15:31,909 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.2 K 2024-11-20T12:15:31,910 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 223b131a87a44f93b2233022b34efc0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732104930217 2024-11-20T12:15:31,910 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f97598a071484b65a6d5d29fb3a7b4d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732104930217 2024-11-20T12:15:31,910 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e32c471a99234a80a21a4e9ab562a33e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104930314 2024-11-20T12:15:31,910 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c74e4aa9f6484e8abb72ec68ca540532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104930314 2024-11-20T12:15:31,911 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 88771729cbd74122ada8edeadd0505f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:31,911 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d352e5be124fa6b1953427031e5d20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:31,927 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:31,928 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/5f683517aac04c1c9d8f3427d5863fee is 50, key is test_row_0/A:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:31,933 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:31,933 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/25af8533860f4103b96ec7170ed7f335 is 50, key is test_row_0/B:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:31,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742078_1254 (size=12104) 2024-11-20T12:15:31,954 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/25af8533860f4103b96ec7170ed7f335 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/25af8533860f4103b96ec7170ed7f335 2024-11-20T12:15:31,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742077_1253 (size=12104) 2024-11-20T12:15:31,964 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 25af8533860f4103b96ec7170ed7f335(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:31,965 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:31,965 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104931908; duration=0sec 2024-11-20T12:15:31,965 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:31,965 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:31,965 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:31,967 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:31,967 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:31,967 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:31,967 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.2 K 2024-11-20T12:15:31,968 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 00772f4b731b48058b5cf1e1043ea150, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732104930217 2024-11-20T12:15:31,968 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d98c035e9249c593c29627abe00fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104930314 2024-11-20T12:15:31,968 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ade35d9f0de6433b9408c238c3e53b6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:31,995 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:31,995 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/c1752abe147b41719a8e84dbdf68d075 is 50, key is test_row_0/C:col10/1732104930657/Put/seqid=0 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104931353, max timestamp=1732104932000Num uniques 859; 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104931353, max timestamp=1732104932000Num uniques 859; 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104931353, max timestamp=1732104932000Num uniques 859; 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:32,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:32,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:32,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:32,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:32,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d065c6288cc74e3abdac0c5f9a5c1086 is 50, key is test_row_0/A:col10/1732104932005/Put/seqid=0 2024-11-20T12:15:32,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742079_1255 (size=12104) 2024-11-20T12:15:32,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742080_1256 (size=14341) 2024-11-20T12:15:32,058 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/c1752abe147b41719a8e84dbdf68d075 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c1752abe147b41719a8e84dbdf68d075 2024-11-20T12:15:32,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104932014, max timestamp=1732104932057Num uniques 859; 2024-11-20T12:15:32,060 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:32,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104932014, max timestamp=1732104932057Num uniques 859; 2024-11-20T12:15:32,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732104932014, max timestamp=1732104932057Num uniques 859; 2024-11-20T12:15:32,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:32,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:15:32,066 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into c1752abe147b41719a8e84dbdf68d075(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:32,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:32,066 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104931908; duration=0sec 2024-11-20T12:15:32,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:32,066 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:32,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104992088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104992088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104992090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104992099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104992105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104992207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104992207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104992207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104992207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104992217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,339 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:15:32,363 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/5f683517aac04c1c9d8f3427d5863fee as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/5f683517aac04c1c9d8f3427d5863fee 2024-11-20T12:15:32,370 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into 5f683517aac04c1c9d8f3427d5863fee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:32,370 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:32,370 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104931908; duration=0sec 2024-11-20T12:15:32,370 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:32,370 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:32,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104992420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104992420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104992424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104992424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104992426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d065c6288cc74e3abdac0c5f9a5c1086 2024-11-20T12:15:32,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4624f55f315e47f8a4b58d0109128a82 is 50, key is test_row_0/B:col10/1732104932005/Put/seqid=0 2024-11-20T12:15:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742081_1257 (size=12001) 2024-11-20T12:15:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:32,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104992737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104992737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104992737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104992747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:32,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104992747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:32,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:32,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:32,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4624f55f315e47f8a4b58d0109128a82 2024-11-20T12:15:32,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/f61f917510934dd5904f14dbbbe72f55 is 50, key is test_row_0/C:col10/1732104932005/Put/seqid=0 2024-11-20T12:15:32,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742082_1258 (size=12001) 2024-11-20T12:15:32,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/f61f917510934dd5904f14dbbbe72f55 2024-11-20T12:15:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d065c6288cc74e3abdac0c5f9a5c1086 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086 2024-11-20T12:15:32,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086, entries=200, sequenceid=88, filesize=14.0 K 2024-11-20T12:15:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4624f55f315e47f8a4b58d0109128a82 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82 2024-11-20T12:15:32,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82, entries=150, sequenceid=88, filesize=11.7 K 2024-11-20T12:15:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/f61f917510934dd5904f14dbbbe72f55 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55 2024-11-20T12:15:32,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55, entries=150, sequenceid=88, filesize=11.7 K 2024-11-20T12:15:32,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for ceaa0a251a1510d6c7c95683bd6968dd in 984ms, sequenceid=88, compaction requested=false 2024-11-20T12:15:32,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:33,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T12:15:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,085 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T12:15:33,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:33,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:33,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c6e2f0d39d164f80becc25e74053593c is 50, key is test_row_0/A:col10/1732104932087/Put/seqid=0 2024-11-20T12:15:33,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742083_1259 (size=12001) 2024-11-20T12:15:33,143 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c6e2f0d39d164f80becc25e74053593c 2024-11-20T12:15:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/b5a600e55c0c4c06babeae0ff1190e90 is 50, key is test_row_0/B:col10/1732104932087/Put/seqid=0 2024-11-20T12:15:33,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742084_1260 (size=12001) 2024-11-20T12:15:33,195 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/b5a600e55c0c4c06babeae0ff1190e90 2024-11-20T12:15:33,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/d3246f3ed83d4de798b2a46ef65809f7 is 50, key is test_row_0/C:col10/1732104932087/Put/seqid=0 2024-11-20T12:15:33,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742085_1261 (size=12001) 2024-11-20T12:15:33,233 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/d3246f3ed83d4de798b2a46ef65809f7 2024-11-20T12:15:33,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/c6e2f0d39d164f80becc25e74053593c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c 2024-11-20T12:15:33,245 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c, entries=150, sequenceid=125, filesize=11.7 K 2024-11-20T12:15:33,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/b5a600e55c0c4c06babeae0ff1190e90 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90 2024-11-20T12:15:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:33,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:33,251 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90, entries=150, sequenceid=125, filesize=11.7 K 2024-11-20T12:15:33,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/d3246f3ed83d4de798b2a46ef65809f7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7 2024-11-20T12:15:33,258 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7, entries=150, sequenceid=125, filesize=11.7 K 2024-11-20T12:15:33,259 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=26.84 KB/27480 for ceaa0a251a1510d6c7c95683bd6968dd in 174ms, sequenceid=125, compaction requested=true 2024-11-20T12:15:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T12:15:33,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T12:15:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:33,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T12:15:33,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-11-20T12:15:33,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:33,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:33,267 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8770 sec 2024-11-20T12:15:33,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a9c45d4137954c16a604fdb0a74da422 is 50, key is test_row_0/A:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:33,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742086_1262 (size=12151) 2024-11-20T12:15:33,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104933283, max timestamp=1732104933291Num uniques 859; 2024-11-20T12:15:33,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104933283, max timestamp=1732104933291Num uniques 859; 2024-11-20T12:15:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104933283, max timestamp=1732104933291Num uniques 859; 2024-11-20T12:15:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:33,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:15:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104993316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104993317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104993319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104993320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104993321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104993436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104993436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104993436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104993437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104993437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T12:15:33,537 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:33,538 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:33,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T12:15:33,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:33,540 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:33,540 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:33,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:33,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104993647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104993647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104993647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104993647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104993647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a9c45d4137954c16a604fdb0a74da422 2024-11-20T12:15:33,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T12:15:33,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:33,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/acd493a3a0984631be3cdcd5e52199d3 is 50, key is test_row_0/B:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742087_1263 (size=12151) 2024-11-20T12:15:33,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/acd493a3a0984631be3cdcd5e52199d3 2024-11-20T12:15:33,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 is 50, key is test_row_0/C:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:33,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742088_1264 (size=12151) 2024-11-20T12:15:33,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T12:15:33,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:33,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:33,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104993958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104993958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104993958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104993958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104993958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T12:15:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 2024-11-20T12:15:34,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a9c45d4137954c16a604fdb0a74da422 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422 2024-11-20T12:15:34,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T12:15:34,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T12:15:34,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/acd493a3a0984631be3cdcd5e52199d3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3 2024-11-20T12:15:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:34,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T12:15:34,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 2024-11-20T12:15:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:34,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53, entries=150, sequenceid=136, filesize=11.9 K 2024-11-20T12:15:34,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ceaa0a251a1510d6c7c95683bd6968dd in 910ms, sequenceid=136, compaction requested=true 2024-11-20T12:15:34,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:34,174 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:34,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:34,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:34,176 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:34,176 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50597 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:34,176 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:34,176 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:34,177 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/5f683517aac04c1c9d8f3427d5863fee, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=49.4 K 2024-11-20T12:15:34,177 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f683517aac04c1c9d8f3427d5863fee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:34,178 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d065c6288cc74e3abdac0c5f9a5c1086, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732104931361 2024-11-20T12:15:34,178 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e2f0d39d164f80becc25e74053593c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732104932087 2024-11-20T12:15:34,179 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:34,179 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:34,179 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,179 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/25af8533860f4103b96ec7170ed7f335, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=47.1 K 2024-11-20T12:15:34,179 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c45d4137954c16a604fdb0a74da422, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:34,179 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 25af8533860f4103b96ec7170ed7f335, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:34,183 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4624f55f315e47f8a4b58d0109128a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732104932003 2024-11-20T12:15:34,184 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a600e55c0c4c06babeae0ff1190e90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732104932087 2024-11-20T12:15:34,185 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting acd493a3a0984631be3cdcd5e52199d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:34,221 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#218 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:34,221 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/55e582d416d549658a80a919fcc15b95 is 50, key is test_row_0/A:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:34,231 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:34,232 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/bc987b797eb54142bbaaa0835696c77f is 50, key is test_row_0/B:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:34,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742089_1265 (size=12391) 2024-11-20T12:15:34,271 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/55e582d416d549658a80a919fcc15b95 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/55e582d416d549658a80a919fcc15b95 2024-11-20T12:15:34,277 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into 55e582d416d549658a80a919fcc15b95(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:34,277 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:34,277 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=12, startTime=1732104934174; duration=0sec 2024-11-20T12:15:34,277 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:34,277 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:34,278 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:34,286 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:34,287 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:34,287 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,287 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c1752abe147b41719a8e84dbdf68d075, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=47.1 K 2024-11-20T12:15:34,288 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1752abe147b41719a8e84dbdf68d075, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1732104930657 2024-11-20T12:15:34,288 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f61f917510934dd5904f14dbbbe72f55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732104932003 2024-11-20T12:15:34,288 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3246f3ed83d4de798b2a46ef65809f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732104932087 2024-11-20T12:15:34,289 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3894e26ef6bf4a1ea89dfcc60b9f5e53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:34,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742090_1266 (size=12391) 2024-11-20T12:15:34,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T12:15:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:34,310 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:34,314 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/bc987b797eb54142bbaaa0835696c77f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/bc987b797eb54142bbaaa0835696c77f 2024-11-20T12:15:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7fa506250e14fdab1becd3cdc48df13 is 50, key is test_row_0/A:col10/1732104933303/Put/seqid=0 2024-11-20T12:15:34,324 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into bc987b797eb54142bbaaa0835696c77f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:34,324 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:34,324 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=12, startTime=1732104934176; duration=0sec 2024-11-20T12:15:34,324 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#220 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:34,324 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:34,324 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:34,324 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/005650713d3e49f09b6bb01f469a2868 is 50, key is test_row_0/C:col10/1732104933262/Put/seqid=0 2024-11-20T12:15:34,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742091_1267 (size=12151) 2024-11-20T12:15:34,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742092_1268 (size=12391) 2024-11-20T12:15:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:34,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104994480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104994481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104994481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104994482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104994487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104994597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104994597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104994598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104994598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104994598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:34,733 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7fa506250e14fdab1becd3cdc48df13 2024-11-20T12:15:34,738 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/005650713d3e49f09b6bb01f469a2868 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/005650713d3e49f09b6bb01f469a2868 2024-11-20T12:15:34,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4b68d9f684be4121a0cf032eb67d7b12 is 50, key is test_row_0/B:col10/1732104933303/Put/seqid=0 2024-11-20T12:15:34,749 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into 005650713d3e49f09b6bb01f469a2868(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:34,749 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:34,749 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=12, startTime=1732104934177; duration=0sec 2024-11-20T12:15:34,749 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:34,749 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:34,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742093_1269 (size=12151) 2024-11-20T12:15:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104994806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104994806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104994806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104994807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:34,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104994807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:34,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104995116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104995116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104995117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104995117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104995117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,161 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4b68d9f684be4121a0cf032eb67d7b12 2024-11-20T12:15:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/17727838ccd74f97b93a0a146a9c2066 is 50, key is test_row_0/C:col10/1732104933303/Put/seqid=0 2024-11-20T12:15:35,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742094_1270 (size=12151) 2024-11-20T12:15:35,194 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/17727838ccd74f97b93a0a146a9c2066 2024-11-20T12:15:35,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7fa506250e14fdab1becd3cdc48df13 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13 2024-11-20T12:15:35,211 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T12:15:35,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4b68d9f684be4121a0cf032eb67d7b12 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12 2024-11-20T12:15:35,215 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T12:15:35,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/17727838ccd74f97b93a0a146a9c2066 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066 2024-11-20T12:15:35,222 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066, entries=150, sequenceid=172, filesize=11.9 K 2024-11-20T12:15:35,227 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ceaa0a251a1510d6c7c95683bd6968dd in 918ms, sequenceid=172, compaction requested=false 2024-11-20T12:15:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:35,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T12:15:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T12:15:35,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T12:15:35,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6880 sec 2024-11-20T12:15:35,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6960 sec 2024-11-20T12:15:35,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:15:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:35,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:35,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7735d02477f4b89ae5818b16884bd0d is 50, key is test_row_0/A:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104995651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742095_1271 (size=12151) 2024-11-20T12:15:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7735d02477f4b89ae5818b16884bd0d 2024-11-20T12:15:35,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104995655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104995656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104995657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104995659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T12:15:35,688 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:35,688 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T12:15:35,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:35,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14f4302630354a3bb93250518b77d210 is 50, key is test_row_0/B:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:35,694 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:35,695 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:35,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:35,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742096_1272 (size=12151) 2024-11-20T12:15:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104995766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104995767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104995767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104995767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104995777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:35,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:35,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:35,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:35,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:35,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:35,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104995977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104995977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104995977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104995977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104995986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:36,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:36,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14f4302630354a3bb93250518b77d210 2024-11-20T12:15:36,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3c5f8031ca8b4bb181cb0452cd45f187 is 50, key is test_row_0/C:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:36,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742097_1273 (size=12151) 2024-11-20T12:15:36,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104996287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104996287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104996287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104996287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104996306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:36,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:36,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:36,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:36,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3c5f8031ca8b4bb181cb0452cd45f187 2024-11-20T12:15:36,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7735d02477f4b89ae5818b16884bd0d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d 2024-11-20T12:15:36,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:15:36,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14f4302630354a3bb93250518b77d210 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210 2024-11-20T12:15:36,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:15:36,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3c5f8031ca8b4bb181cb0452cd45f187 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187 2024-11-20T12:15:36,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:15:36,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ceaa0a251a1510d6c7c95683bd6968dd in 952ms, sequenceid=189, compaction requested=true 2024-11-20T12:15:36,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:36,584 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:36,584 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:36,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:36,586 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:36,586 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:36,586 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,586 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:36,586 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:36,586 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/55e582d416d549658a80a919fcc15b95, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.8 K 2024-11-20T12:15:36,586 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,586 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/bc987b797eb54142bbaaa0835696c77f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.8 K 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e582d416d549658a80a919fcc15b95, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d7fa506250e14fdab1becd3cdc48df13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732104933303 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc987b797eb54142bbaaa0835696c77f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f7735d02477f4b89ae5818b16884bd0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b68d9f684be4121a0cf032eb67d7b12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732104933303 2024-11-20T12:15:36,587 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14f4302630354a3bb93250518b77d210, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:36,604 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:36,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/73f5d9025e0e4ea7b628219db904f89d is 50, key is test_row_0/B:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:36,607 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:36,608 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8397489a4d004822803e91a56d150eb9 is 50, key is test_row_0/A:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:36,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,615 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:36,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7612d89180f42aa84f665c94473de66 is 50, key is test_row_0/A:col10/1732104935655/Put/seqid=0 2024-11-20T12:15:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742099_1275 (size=12493) 2024-11-20T12:15:36,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742098_1274 (size=12493) 2024-11-20T12:15:36,663 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/73f5d9025e0e4ea7b628219db904f89d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/73f5d9025e0e4ea7b628219db904f89d 2024-11-20T12:15:36,665 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8397489a4d004822803e91a56d150eb9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8397489a4d004822803e91a56d150eb9 2024-11-20T12:15:36,671 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 73f5d9025e0e4ea7b628219db904f89d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:36,671 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:36,671 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104936584; duration=0sec 2024-11-20T12:15:36,671 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:36,672 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:36,672 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:36,673 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:36,673 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:36,673 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:36,673 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/005650713d3e49f09b6bb01f469a2868, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.8 K 2024-11-20T12:15:36,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005650713d3e49f09b6bb01f469a2868, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732104933261 2024-11-20T12:15:36,674 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17727838ccd74f97b93a0a146a9c2066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732104933303 2024-11-20T12:15:36,674 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into 8397489a4d004822803e91a56d150eb9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:36,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:36,674 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104936583; duration=0sec 2024-11-20T12:15:36,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:36,675 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:36,675 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c5f8031ca8b4bb181cb0452cd45f187, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:36,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742100_1276 (size=12151) 2024-11-20T12:15:36,696 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:36,696 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/872ea7d7720a474d95baef88884636a2 is 50, key is test_row_0/C:col10/1732104935629/Put/seqid=0 2024-11-20T12:15:36,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742101_1277 (size=12493) 2024-11-20T12:15:36,737 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/872ea7d7720a474d95baef88884636a2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/872ea7d7720a474d95baef88884636a2 2024-11-20T12:15:36,744 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into 872ea7d7720a474d95baef88884636a2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:36,744 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:36,744 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104936584; duration=0sec 2024-11-20T12:15:36,744 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:36,744 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:36,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:36,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104996812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104996814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104996816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104996816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104996815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:36,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104996927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104996927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104996927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:36,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:36,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104996932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:36,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,091 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7612d89180f42aa84f665c94473de66 2024-11-20T12:15:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/62548bdbf35b4f54929cc17753470dd6 is 50, key is test_row_0/B:col10/1732104935655/Put/seqid=0 2024-11-20T12:15:37,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742102_1278 (size=12151) 2024-11-20T12:15:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104997136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104997136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104997136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104997136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104997447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104997447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104997447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104997447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,504 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/62548bdbf35b4f54929cc17753470dd6 2024-11-20T12:15:37,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/98cc0e07987c4ed7b3bb4432dd6f63c6 is 50, key is test_row_0/C:col10/1732104935655/Put/seqid=0 2024-11-20T12:15:37,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742103_1279 (size=12151) 2024-11-20T12:15:37,515 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/98cc0e07987c4ed7b3bb4432dd6f63c6 2024-11-20T12:15:37,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a7612d89180f42aa84f665c94473de66 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66 2024-11-20T12:15:37,525 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T12:15:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/62548bdbf35b4f54929cc17753470dd6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6 2024-11-20T12:15:37,531 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T12:15:37,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/98cc0e07987c4ed7b3bb4432dd6f63c6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6 2024-11-20T12:15:37,535 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6, entries=150, sequenceid=206, filesize=11.9 K 2024-11-20T12:15:37,536 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ceaa0a251a1510d6c7c95683bd6968dd in 921ms, sequenceid=206, compaction requested=false 2024-11-20T12:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:37,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T12:15:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T12:15:37,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T12:15:37,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-11-20T12:15:37,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8520 sec 2024-11-20T12:15:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:37,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:15:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:37,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:37,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:37,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/42d7475f0e1b4d5ca5c375a3f503460b is 50, key is test_row_0/A:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T12:15:37,838 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:37,838 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T12:15:37,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T12:15:37,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:37,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:37,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742104_1280 (size=14541) 2024-11-20T12:15:37,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104997896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T12:15:37,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104997957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104997957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104997957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104997957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:37,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T12:15:37,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:37,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104998007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T12:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T12:15:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104998217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/42d7475f0e1b4d5ca5c375a3f503460b 2024-11-20T12:15:38,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/dbea1b756cfe412c82ca304cfbc33dc8 is 50, key is test_row_0/B:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:38,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742105_1281 (size=12151) 2024-11-20T12:15:38,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/dbea1b756cfe412c82ca304cfbc33dc8 2024-11-20T12:15:38,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4c050fc535d14eb1a70e52946641d6ba is 50, key is test_row_0/C:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:38,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742106_1282 (size=12151) 2024-11-20T12:15:38,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4c050fc535d14eb1a70e52946641d6ba 2024-11-20T12:15:38,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/42d7475f0e1b4d5ca5c375a3f503460b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b 2024-11-20T12:15:38,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b, entries=200, sequenceid=225, filesize=14.2 K 2024-11-20T12:15:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/dbea1b756cfe412c82ca304cfbc33dc8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8 2024-11-20T12:15:38,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T12:15:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:15:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:38,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4c050fc535d14eb1a70e52946641d6ba as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba 2024-11-20T12:15:38,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:15:38,312 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ceaa0a251a1510d6c7c95683bd6968dd in 481ms, sequenceid=225, compaction requested=true 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:38,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:38,312 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:38,312 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:38,313 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,313 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,313 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/73f5d9025e0e4ea7b628219db904f89d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.9 K 2024-11-20T12:15:38,313 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8397489a4d004822803e91a56d150eb9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=38.3 K 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8397489a4d004822803e91a56d150eb9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:38,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 73f5d9025e0e4ea7b628219db904f89d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:38,314 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7612d89180f42aa84f665c94473de66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732104935653 2024-11-20T12:15:38,314 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 62548bdbf35b4f54929cc17753470dd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732104935653 2024-11-20T12:15:38,314 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42d7475f0e1b4d5ca5c375a3f503460b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936810 2024-11-20T12:15:38,314 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting dbea1b756cfe412c82ca304cfbc33dc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936813 2024-11-20T12:15:38,328 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#236 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:38,329 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/6065621f53134dd3aa7c04531d9b4a72 is 50, key is test_row_0/B:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:38,339 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:38,339 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f9598f9b2bca4e1ab94586ca8ccbb26c is 50, key is test_row_0/A:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:38,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742107_1283 (size=12595) 2024-11-20T12:15:38,350 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/6065621f53134dd3aa7c04531d9b4a72 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/6065621f53134dd3aa7c04531d9b4a72 2024-11-20T12:15:38,356 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 6065621f53134dd3aa7c04531d9b4a72(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:38,356 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:38,356 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104938312; duration=0sec 2024-11-20T12:15:38,356 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:38,356 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:38,356 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:38,357 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:38,357 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:38,358 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,358 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/872ea7d7720a474d95baef88884636a2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=35.9 K 2024-11-20T12:15:38,361 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 872ea7d7720a474d95baef88884636a2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104935629 2024-11-20T12:15:38,361 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 98cc0e07987c4ed7b3bb4432dd6f63c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732104935653 2024-11-20T12:15:38,362 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c050fc535d14eb1a70e52946641d6ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936813 2024-11-20T12:15:38,369 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:38,370 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/1e7a5374dbb148fd9e1538efeec80c4c is 50, key is test_row_0/C:col10/1732104937829/Put/seqid=0 2024-11-20T12:15:38,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742108_1284 (size=12595) 2024-11-20T12:15:38,384 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f9598f9b2bca4e1ab94586ca8ccbb26c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f9598f9b2bca4e1ab94586ca8ccbb26c 2024-11-20T12:15:38,391 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into f9598f9b2bca4e1ab94586ca8ccbb26c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:38,391 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:38,391 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104938312; duration=0sec 2024-11-20T12:15:38,391 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:38,391 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:38,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742109_1285 (size=12595) 2024-11-20T12:15:38,404 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/1e7a5374dbb148fd9e1538efeec80c4c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/1e7a5374dbb148fd9e1538efeec80c4c 2024-11-20T12:15:38,413 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into 1e7a5374dbb148fd9e1538efeec80c4c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:38,413 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:38,413 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104938312; duration=0sec 2024-11-20T12:15:38,413 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:38,413 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:38,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T12:15:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,457 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T12:15:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b085681c38f841219256dc730c7a69f6 is 50, key is test_row_0/A:col10/1732104937889/Put/seqid=0 2024-11-20T12:15:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T12:15:38,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742110_1286 (size=12151) 2024-11-20T12:15:38,489 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b085681c38f841219256dc730c7a69f6 2024-11-20T12:15:38,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e7870767aa7446cb85003f1a2809ec64 is 50, key is test_row_0/B:col10/1732104937889/Put/seqid=0 2024-11-20T12:15:38,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742111_1287 (size=12151) 2024-11-20T12:15:38,521 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e7870767aa7446cb85003f1a2809ec64 2024-11-20T12:15:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/692beb2aa2124baba1568038d46e77e9 is 50, key is test_row_0/C:col10/1732104937889/Put/seqid=0 2024-11-20T12:15:38,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:38,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:38,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742112_1288 (size=12151) 2024-11-20T12:15:38,555 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/692beb2aa2124baba1568038d46e77e9 2024-11-20T12:15:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b085681c38f841219256dc730c7a69f6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6 2024-11-20T12:15:38,566 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6, entries=150, sequenceid=244, filesize=11.9 K 2024-11-20T12:15:38,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e7870767aa7446cb85003f1a2809ec64 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64 2024-11-20T12:15:38,578 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64, entries=150, sequenceid=244, filesize=11.9 K 2024-11-20T12:15:38,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/692beb2aa2124baba1568038d46e77e9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9 2024-11-20T12:15:38,587 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9, entries=150, sequenceid=244, filesize=11.9 K 2024-11-20T12:15:38,588 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=67.09 KB/68700 for ceaa0a251a1510d6c7c95683bd6968dd in 131ms, sequenceid=244, compaction requested=false 2024-11-20T12:15:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T12:15:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:38,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:15:38,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T12:15:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:38,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:38,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T12:15:38,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 745 msec 2024-11-20T12:15:38,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/7571e32b9fb84742a32508f03f3dc90e is 50, key is test_row_0/A:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:38,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 755 msec 2024-11-20T12:15:38,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742113_1289 (size=14591) 2024-11-20T12:15:38,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104998665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104998776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732104998966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,969 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T12:15:38,977 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:38,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T12:15:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T12:15:38,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:38,980 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:38,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732104998977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732104998977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732104998977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,981 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,981 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,982 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104998986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:38,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:39,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/7571e32b9fb84742a32508f03f3dc90e 2024-11-20T12:15:39,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4917c56cfd6e48439cf25c4af5a159b2 is 50, key is test_row_0/B:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:39,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742114_1290 (size=12201) 2024-11-20T12:15:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T12:15:39,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:39,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T12:15:39,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104999297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:39,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:39,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4917c56cfd6e48439cf25c4af5a159b2 2024-11-20T12:15:39,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/43e88fdd75e24123b52c3d696f671606 is 50, key is test_row_0/C:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:39,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:39,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742115_1291 (size=12201) 2024-11-20T12:15:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T12:15:39,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:39,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732104999806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:39,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:39,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/43e88fdd75e24123b52c3d696f671606 2024-11-20T12:15:39,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/7571e32b9fb84742a32508f03f3dc90e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e 2024-11-20T12:15:39,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e, entries=200, sequenceid=258, filesize=14.2 K 2024-11-20T12:15:39,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/4917c56cfd6e48439cf25c4af5a159b2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2 2024-11-20T12:15:39,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T12:15:39,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/43e88fdd75e24123b52c3d696f671606 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606 2024-11-20T12:15:39,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606, entries=150, sequenceid=258, filesize=11.9 K 2024-11-20T12:15:39,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ceaa0a251a1510d6c7c95683bd6968dd in 1281ms, sequenceid=258, compaction requested=true 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:39,869 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:39,869 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:39,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:39,870 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:39,870 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:39,870 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,870 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/6065621f53134dd3aa7c04531d9b4a72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.1 K 2024-11-20T12:15:39,871 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39337 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:39,871 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6065621f53134dd3aa7c04531d9b4a72, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936813 2024-11-20T12:15:39,871 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:39,871 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,871 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f9598f9b2bca4e1ab94586ca8ccbb26c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=38.4 K 2024-11-20T12:15:39,871 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e7870767aa7446cb85003f1a2809ec64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732104937880 2024-11-20T12:15:39,871 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9598f9b2bca4e1ab94586ca8ccbb26c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936813 2024-11-20T12:15:39,873 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4917c56cfd6e48439cf25c4af5a159b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:39,873 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b085681c38f841219256dc730c7a69f6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732104937880 2024-11-20T12:15:39,874 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7571e32b9fb84742a32508f03f3dc90e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:39,882 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:39,882 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e4eb7b8fb44c42758a5bb268d1f2523d is 50, key is test_row_0/B:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:39,883 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:39,884 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a3e0546cffc440c4bbfb257510fdf708 is 50, key is test_row_0/A:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742116_1292 (size=12747) 2024-11-20T12:15:39,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T12:15:39,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,902 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:39,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:39,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a4ab046e7a56487a9256580538d4b2b3 is 50, key is test_row_0/A:col10/1732104938640/Put/seqid=0 2024-11-20T12:15:39,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742117_1293 (size=12747) 2024-11-20T12:15:39,930 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a3e0546cffc440c4bbfb257510fdf708 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a3e0546cffc440c4bbfb257510fdf708 2024-11-20T12:15:39,937 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into a3e0546cffc440c4bbfb257510fdf708(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:39,937 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:39,937 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104939869; duration=0sec 2024-11-20T12:15:39,937 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:39,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:39,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:39,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:39,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:39,939 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:39,939 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/1e7a5374dbb148fd9e1538efeec80c4c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.1 K 2024-11-20T12:15:39,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7a5374dbb148fd9e1538efeec80c4c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104936813 2024-11-20T12:15:39,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692beb2aa2124baba1568038d46e77e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732104937880 2024-11-20T12:15:39,940 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e88fdd75e24123b52c3d696f671606, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:39,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742118_1294 (size=12301) 2024-11-20T12:15:39,955 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a4ab046e7a56487a9256580538d4b2b3 2024-11-20T12:15:39,963 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#248 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:39,963 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/6f1d15c5851c45f3b1c41cda3d067548 is 50, key is test_row_0/C:col10/1732104938547/Put/seqid=0 2024-11-20T12:15:39,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/833b4e131eda4bc2b0ea3ea3443ab32a is 50, key is test_row_0/B:col10/1732104938640/Put/seqid=0 2024-11-20T12:15:39,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742119_1295 (size=12301) 2024-11-20T12:15:39,997 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/833b4e131eda4bc2b0ea3ea3443ab32a 2024-11-20T12:15:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/c52991e7e4314b539a48e9b760408a89 is 50, key is test_row_0/C:col10/1732104938640/Put/seqid=0 2024-11-20T12:15:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742120_1296 (size=12747) 2024-11-20T12:15:40,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742121_1297 (size=12301) 2024-11-20T12:15:40,038 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/c52991e7e4314b539a48e9b760408a89 2024-11-20T12:15:40,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/a4ab046e7a56487a9256580538d4b2b3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3 2024-11-20T12:15:40,047 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T12:15:40,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/833b4e131eda4bc2b0ea3ea3443ab32a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a 2024-11-20T12:15:40,054 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T12:15:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/c52991e7e4314b539a48e9b760408a89 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89 2024-11-20T12:15:40,061 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89, entries=150, sequenceid=276, filesize=12.0 K 2024-11-20T12:15:40,062 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for ceaa0a251a1510d6c7c95683bd6968dd in 160ms, sequenceid=276, compaction requested=false 2024-11-20T12:15:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T12:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T12:15:40,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T12:15:40,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-20T12:15:40,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0900 sec 2024-11-20T12:15:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T12:15:40,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:40,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T12:15:40,120 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T12:15:40,121 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:40,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:40,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T12:15:40,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T12:15:40,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:40,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T12:15:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T12:15:40,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T12:15:40,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 158 msec 2024-11-20T12:15:40,283 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 164 msec 2024-11-20T12:15:40,304 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/e4eb7b8fb44c42758a5bb268d1f2523d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e4eb7b8fb44c42758a5bb268d1f2523d 2024-11-20T12:15:40,311 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into e4eb7b8fb44c42758a5bb268d1f2523d(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:40,311 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:40,311 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104939869; duration=0sec 2024-11-20T12:15:40,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:40,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:40,422 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/6f1d15c5851c45f3b1c41cda3d067548 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6f1d15c5851c45f3b1c41cda3d067548 2024-11-20T12:15:40,427 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into 6f1d15c5851c45f3b1c41cda3d067548(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:40,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:40,427 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104939869; duration=0sec 2024-11-20T12:15:40,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:40,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T12:15:40,437 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:40,437 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-20T12:15:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T12:15:40,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:40,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:40,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T12:15:40,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-20T12:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-20T12:15:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-20T12:15:40,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T12:15:40,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-20T12:15:40,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 159 msec 2024-11-20T12:15:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T12:15:40,756 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:40,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-20T12:15:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:40,759 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:40,760 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:40,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:40,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:40,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:40,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:40,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:40,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:40,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:40,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:40,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b2e1aa53b36449839c91e37431edd477 is 50, key is test_row_0/A:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:40,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742122_1298 (size=12301) 2024-11-20T12:15:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:40,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b2e1aa53b36449839c91e37431edd477 2024-11-20T12:15:40,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3c3a5257b6b9484982e9a0cb8cfa6c74 is 50, key is test_row_0/B:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:40,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:40,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:40,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:40,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742123_1299 (size=12301) 2024-11-20T12:15:40,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3c3a5257b6b9484982e9a0cb8cfa6c74 2024-11-20T12:15:40,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/cc81aa456c3b45caa549d6efd669fced is 50, key is test_row_0/C:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:40,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104940936, max timestamp=1732104940977Num uniques 859; 2024-11-20T12:15:40,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104940936, max timestamp=1732104940977Num uniques 859; 2024-11-20T12:15:40,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:40,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:40,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742124_1300 (size=12301) 2024-11-20T12:15:40,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732104940936, max timestamp=1732104940977Num uniques 859; 2024-11-20T12:15:40,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T12:15:40,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/cc81aa456c3b45caa549d6efd669fced 2024-11-20T12:15:40,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/b2e1aa53b36449839c91e37431edd477 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477 2024-11-20T12:15:41,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T12:15:41,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3c3a5257b6b9484982e9a0cb8cfa6c74 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74 2024-11-20T12:15:41,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T12:15:41,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/cc81aa456c3b45caa549d6efd669fced as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced 2024-11-20T12:15:41,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced, entries=150, sequenceid=291, filesize=12.0 K 2024-11-20T12:15:41,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.24 KB/102645 for ceaa0a251a1510d6c7c95683bd6968dd in 171ms, sequenceid=291, compaction requested=true 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:41,015 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:41,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T12:15:41,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:41,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=106.95 KB heapSize=304.38 KB 2024-11-20T12:15:41,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:41,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:41,017 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,017 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a3e0546cffc440c4bbfb257510fdf708, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.5 K 2024-11-20T12:15:41,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e0546cffc440c4bbfb257510fdf708, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:41,018 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:41,018 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a4ab046e7a56487a9256580538d4b2b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732104938640 2024-11-20T12:15:41,018 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:41,018 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,018 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e4eb7b8fb44c42758a5bb268d1f2523d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.5 K 2024-11-20T12:15:41,018 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e1aa53b36449839c91e37431edd477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:41,018 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4eb7b8fb44c42758a5bb268d1f2523d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:41,019 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833b4e131eda4bc2b0ea3ea3443ab32a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732104938640 2024-11-20T12:15:41,019 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c3a5257b6b9484982e9a0cb8cfa6c74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:41,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:41,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/2a4b9802ec9f4b49b6a6e61b126257f9 is 50, key is test_row_0/A:col10/1732104941016/Put/seqid=0 2024-11-20T12:15:41,042 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105001036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,045 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/6c3118a1e89c421997cd278e67baf93c is 50, key is test_row_0/A:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:41,047 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:41,048 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3eaee48214524b2b8b683ec11eaf5f83 is 50, key is test_row_0/B:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105001041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105001043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105001044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105001044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742125_1301 (size=17181) 2024-11-20T12:15:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742126_1302 (size=12949) 2024-11-20T12:15:41,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742127_1303 (size=12949) 2024-11-20T12:15:41,090 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3eaee48214524b2b8b683ec11eaf5f83 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3eaee48214524b2b8b683ec11eaf5f83 2024-11-20T12:15:41,095 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 3eaee48214524b2b8b683ec11eaf5f83(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:41,095 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:41,095 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104941015; duration=0sec 2024-11-20T12:15:41,095 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:41,095 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:41,095 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:41,096 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37349 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:41,097 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:41,097 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,097 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6f1d15c5851c45f3b1c41cda3d067548, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.5 K 2024-11-20T12:15:41,097 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f1d15c5851c45f3b1c41cda3d067548, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1732104938547 2024-11-20T12:15:41,097 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52991e7e4314b539a48e9b760408a89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732104938640 2024-11-20T12:15:41,098 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc81aa456c3b45caa549d6efd669fced, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:41,110 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:41,111 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/e18fbe7dd2e14d0ba63bc611ac19cdf5 is 50, key is test_row_0/C:col10/1732104940836/Put/seqid=0 2024-11-20T12:15:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742128_1304 (size=12949) 2024-11-20T12:15:41,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105001147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,154 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/e18fbe7dd2e14d0ba63bc611ac19cdf5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e18fbe7dd2e14d0ba63bc611ac19cdf5 2024-11-20T12:15:41,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105001157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105001157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105001157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105001157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,162 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into e18fbe7dd2e14d0ba63bc611ac19cdf5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:41,162 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:41,162 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104941015; duration=0sec 2024-11-20T12:15:41,162 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:41,162 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:41,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105001356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105001368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105001368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105001369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105001369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:41,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/2a4b9802ec9f4b49b6a6e61b126257f9 2024-11-20T12:15:41,488 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/6c3118a1e89c421997cd278e67baf93c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/6c3118a1e89c421997cd278e67baf93c 2024-11-20T12:15:41,493 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into 6c3118a1e89c421997cd278e67baf93c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:41,493 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:41,493 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104941015; duration=0sec 2024-11-20T12:15:41,493 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:41,493 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:41,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/10bb07ec8b864cbca369d7e89d9f147b is 50, key is test_row_0/B:col10/1732104941016/Put/seqid=0 2024-11-20T12:15:41,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742129_1305 (size=12301) 2024-11-20T12:15:41,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105001667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105001676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105001676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105001676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105001676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:41,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:41,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/10bb07ec8b864cbca369d7e89d9f147b 2024-11-20T12:15:41,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b is 50, key is test_row_0/C:col10/1732104941016/Put/seqid=0 2024-11-20T12:15:41,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742130_1306 (size=12301) 2024-11-20T12:15:41,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:41,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:41,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:42,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105002177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105002186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105002186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105002186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105002186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:42,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:42,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:42,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:42,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b 2024-11-20T12:15:42,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/2a4b9802ec9f4b49b6a6e61b126257f9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9 2024-11-20T12:15:42,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9, entries=250, sequenceid=326, filesize=16.8 K 2024-11-20T12:15:42,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/10bb07ec8b864cbca369d7e89d9f147b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b 2024-11-20T12:15:42,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T12:15:42,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b 2024-11-20T12:15:42,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T12:15:42,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.37 KB/123255, heapSize ~344.18 KB/352440, currentSize=67.09 KB/68700 for ceaa0a251a1510d6c7c95683bd6968dd in 1314ms, sequenceid=326, compaction requested=false 2024-11-20T12:15:42,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:42,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-20T12:15:42,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,447 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:42,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:42,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 is 50, key is test_row_0/A:col10/1732104941032/Put/seqid=0 2024-11-20T12:15:42,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742131_1307 (size=12301) 2024-11-20T12:15:42,462 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 2024-11-20T12:15:42,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14fab086c893464a846a6789f1714142 is 50, key is test_row_0/B:col10/1732104941032/Put/seqid=0 2024-11-20T12:15:42,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742132_1308 (size=12301) 2024-11-20T12:15:42,484 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14fab086c893464a846a6789f1714142 2024-11-20T12:15:42,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dbf4e4bac732414cbeb5a2a1c3349bed is 50, key is test_row_0/C:col10/1732104941032/Put/seqid=0 2024-11-20T12:15:42,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742133_1309 (size=12301) 2024-11-20T12:15:42,528 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dbf4e4bac732414cbeb5a2a1c3349bed 2024-11-20T12:15:42,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 2024-11-20T12:15:42,538 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T12:15:42,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/14fab086c893464a846a6789f1714142 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142 2024-11-20T12:15:42,545 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T12:15:42,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dbf4e4bac732414cbeb5a2a1c3349bed as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed 2024-11-20T12:15:42,552 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed, entries=150, sequenceid=342, filesize=12.0 K 2024-11-20T12:15:42,554 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for ceaa0a251a1510d6c7c95683bd6968dd in 108ms, sequenceid=342, compaction requested=true 2024-11-20T12:15:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-20T12:15:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-20T12:15:42,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T12:15:42,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7950 sec 2024-11-20T12:15:42,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8030 sec 2024-11-20T12:15:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-20T12:15:42,917 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:42,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-20T12:15:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T12:15:42,919 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:42,920 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:42,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T12:15:43,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-20T12:15:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-20T12:15:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-20T12:15:43,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-20T12:15:43,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-20T12:15:43,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 158 msec 2024-11-20T12:15:43,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:43,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:43,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:43,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8379ba3a285640d1963d6bffa0322798 is 50, key is test_row_0/A:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:43,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742134_1310 (size=14741) 2024-11-20T12:15:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T12:15:43,236 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:43,237 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-20T12:15:43,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:43,238 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:43,239 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:43,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105003248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105003248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105003249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105003249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105003253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,258 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:43,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105003356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105003356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105003356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105003356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105003367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105003566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105003566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105003567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105003567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105003577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8379ba3a285640d1963d6bffa0322798 2024-11-20T12:15:43,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/758c6d372151477487987838f5470dca is 50, key is test_row_0/B:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:43,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742135_1311 (size=12301) 2024-11-20T12:15:43,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:43,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:43,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:43,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105003876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105003877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:43,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105003887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/758c6d372151477487987838f5470dca 2024-11-20T12:15:44,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/6d21832d8e2a487ea1d795f1134efd27 is 50, key is test_row_0/C:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:44,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742136_1312 (size=12301) 2024-11-20T12:15:44,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:44,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:44,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:44,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105004386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105004386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105004386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105004397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105004397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:44,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:44,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/6d21832d8e2a487ea1d795f1134efd27 2024-11-20T12:15:44,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8379ba3a285640d1963d6bffa0322798 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798 2024-11-20T12:15:44,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798, entries=200, sequenceid=355, filesize=14.4 K 2024-11-20T12:15:44,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/758c6d372151477487987838f5470dca as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca 2024-11-20T12:15:44,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca, entries=150, sequenceid=355, filesize=12.0 K 2024-11-20T12:15:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/6d21832d8e2a487ea1d795f1134efd27 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27 2024-11-20T12:15:44,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27, entries=150, sequenceid=355, filesize=12.0 K 2024-11-20T12:15:44,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ceaa0a251a1510d6c7c95683bd6968dd in 1254ms, sequenceid=355, compaction requested=true 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:44,481 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:44,481 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:44,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:44,485 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:44,485 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:44,485 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,485 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3eaee48214524b2b8b683ec11eaf5f83, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=48.7 K 2024-11-20T12:15:44,485 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:44,485 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:44,486 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,486 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/6c3118a1e89c421997cd278e67baf93c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=55.8 K 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eaee48214524b2b8b683ec11eaf5f83, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c3118a1e89c421997cd278e67baf93c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4b9802ec9f4b49b6a6e61b126257f9, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732104940977 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 10bb07ec8b864cbca369d7e89d9f147b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732104940977 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e71b0ee654a44a6bbff5a6e86a1a1f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732104941030 2024-11-20T12:15:44,486 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 14fab086c893464a846a6789f1714142, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732104941030 2024-11-20T12:15:44,487 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8379ba3a285640d1963d6bffa0322798, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943190 2024-11-20T12:15:44,487 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 758c6d372151477487987838f5470dca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943192 2024-11-20T12:15:44,511 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:44,511 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:44,511 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3db3189b53cd47858c191935be5c360e is 50, key is test_row_0/B:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:44,511 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/bc245862ee6945b38b34cf8a4782b6bb is 50, key is test_row_0/A:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742137_1313 (size=13085) 2024-11-20T12:15:44,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742138_1314 (size=13085) 2024-11-20T12:15:44,531 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/3db3189b53cd47858c191935be5c360e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3db3189b53cd47858c191935be5c360e 2024-11-20T12:15:44,538 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/bc245862ee6945b38b34cf8a4782b6bb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/bc245862ee6945b38b34cf8a4782b6bb 2024-11-20T12:15:44,540 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 3db3189b53cd47858c191935be5c360e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:44,540 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:44,540 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=12, startTime=1732104944481; duration=0sec 2024-11-20T12:15:44,540 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:44,540 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:44,541 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:44,544 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:44,544 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:44,544 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,544 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e18fbe7dd2e14d0ba63bc611ac19cdf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=48.7 K 2024-11-20T12:15:44,545 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e18fbe7dd2e14d0ba63bc611ac19cdf5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732104940836 2024-11-20T12:15:44,546 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a6cfb55f530f4ea79d8f9eb7cbfd4a9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732104940977 2024-11-20T12:15:44,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf4e4bac732414cbeb5a2a1c3349bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732104941030 2024-11-20T12:15:44,547 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d21832d8e2a487ea1d795f1134efd27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943192 2024-11-20T12:15:44,548 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into bc245862ee6945b38b34cf8a4782b6bb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:44,548 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:44,548 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=12, startTime=1732104944481; duration=0sec 2024-11-20T12:15:44,548 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:44,548 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:44,556 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:44,557 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/b24655cdeb524f0d90cfab8a392fd1c8 is 50, key is test_row_0/C:col10/1732104943192/Put/seqid=0 2024-11-20T12:15:44,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742139_1315 (size=13085) 2024-11-20T12:15:44,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-20T12:15:44,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:44,614 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:44,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:44,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/4e16e60ab2c14a6f91279a791e9cec8e is 50, key is test_row_0/A:col10/1732104943248/Put/seqid=0 2024-11-20T12:15:44,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742140_1316 (size=12301) 2024-11-20T12:15:44,968 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/b24655cdeb524f0d90cfab8a392fd1c8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/b24655cdeb524f0d90cfab8a392fd1c8 2024-11-20T12:15:44,974 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into b24655cdeb524f0d90cfab8a392fd1c8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:44,974 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:44,974 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=12, startTime=1732104944481; duration=0sec 2024-11-20T12:15:44,974 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:44,974 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:45,027 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/4e16e60ab2c14a6f91279a791e9cec8e 2024-11-20T12:15:45,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/8ccaa5fc8f034791a5bf859ea6810cd0 is 50, key is test_row_0/B:col10/1732104943248/Put/seqid=0 2024-11-20T12:15:45,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742141_1317 (size=12301) 2024-11-20T12:15:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:45,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:45,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:45,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105005406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105005407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,410 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105005409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105005410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105005410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,415 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,442 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/8ccaa5fc8f034791a5bf859ea6810cd0 2024-11-20T12:15:45,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a4d9bbeba82c43f8b43a691781b60b4b is 50, key is test_row_0/C:col10/1732104943248/Put/seqid=0 2024-11-20T12:15:45,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742142_1318 (size=12301) 2024-11-20T12:15:45,453 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a4d9bbeba82c43f8b43a691781b60b4b 2024-11-20T12:15:45,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/4e16e60ab2c14a6f91279a791e9cec8e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e 2024-11-20T12:15:45,462 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T12:15:45,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/8ccaa5fc8f034791a5bf859ea6810cd0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0 2024-11-20T12:15:45,467 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T12:15:45,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a4d9bbeba82c43f8b43a691781b60b4b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b 2024-11-20T12:15:45,472 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b, entries=150, sequenceid=377, filesize=12.0 K 2024-11-20T12:15:45,473 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for ceaa0a251a1510d6c7c95683bd6968dd in 859ms, sequenceid=377, compaction requested=false 2024-11-20T12:15:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-20T12:15:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-20T12:15:45,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T12:15:45,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-20T12:15:45,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.2460 sec 2024-11-20T12:15:45,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:45,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:45,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:45,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/280bda72a0e2432e8ba8b2c58ea5a9bc is 50, key is test_row_0/A:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:45,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742143_1319 (size=14741) 2024-11-20T12:15:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105005574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105005573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105005575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105005577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105005686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105005686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105005686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105005687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105005896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105005896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105005896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105005897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:45,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/280bda72a0e2432e8ba8b2c58ea5a9bc 2024-11-20T12:15:45,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c6108f2971a7423c8486dbdc88a476da is 50, key is test_row_0/B:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742144_1320 (size=12301) 2024-11-20T12:15:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105006207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105006207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105006207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105006207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c6108f2971a7423c8486dbdc88a476da 2024-11-20T12:15:46,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dc480c5f91df4e609c43a9b286d73981 is 50, key is test_row_0/C:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:46,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742145_1321 (size=12301) 2024-11-20T12:15:46,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dc480c5f91df4e609c43a9b286d73981 2024-11-20T12:15:46,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/280bda72a0e2432e8ba8b2c58ea5a9bc as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc 2024-11-20T12:15:46,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc, entries=200, sequenceid=392, filesize=14.4 K 2024-11-20T12:15:46,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c6108f2971a7423c8486dbdc88a476da as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da 2024-11-20T12:15:46,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da, entries=150, sequenceid=392, filesize=12.0 K 2024-11-20T12:15:46,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/dc480c5f91df4e609c43a9b286d73981 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981 2024-11-20T12:15:46,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981, entries=150, sequenceid=392, filesize=12.0 K 2024-11-20T12:15:46,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ceaa0a251a1510d6c7c95683bd6968dd in 857ms, sequenceid=392, compaction requested=true 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:46,375 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:46,375 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:46,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:46,376 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:46,376 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:46,376 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:46,376 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:46,376 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:46,376 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:46,376 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3db3189b53cd47858c191935be5c360e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.8 K 2024-11-20T12:15:46,376 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/bc245862ee6945b38b34cf8a4782b6bb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=39.2 K 2024-11-20T12:15:46,377 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc245862ee6945b38b34cf8a4782b6bb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943192 2024-11-20T12:15:46,377 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3db3189b53cd47858c191935be5c360e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943192 2024-11-20T12:15:46,377 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccaa5fc8f034791a5bf859ea6810cd0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732104943247 2024-11-20T12:15:46,377 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e16e60ab2c14a6f91279a791e9cec8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732104943247 2024-11-20T12:15:46,377 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c6108f2971a7423c8486dbdc88a476da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:46,378 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 280bda72a0e2432e8ba8b2c58ea5a9bc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:46,387 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:46,388 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/02838794ca424c31a129c354025cc035 is 50, key is test_row_0/B:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:46,391 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:46,391 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/120f9cb0575b4d6c8382962ce2702189 is 50, key is test_row_0/A:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:46,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742146_1322 (size=13187) 2024-11-20T12:15:46,409 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/02838794ca424c31a129c354025cc035 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/02838794ca424c31a129c354025cc035 2024-11-20T12:15:46,417 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 02838794ca424c31a129c354025cc035(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:46,417 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:46,417 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=13, startTime=1732104946375; duration=0sec 2024-11-20T12:15:46,417 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:46,417 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:46,417 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:46,421 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:46,421 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:46,421 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:46,421 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/b24655cdeb524f0d90cfab8a392fd1c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=36.8 K 2024-11-20T12:15:46,421 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b24655cdeb524f0d90cfab8a392fd1c8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732104943192 2024-11-20T12:15:46,421 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a4d9bbeba82c43f8b43a691781b60b4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732104943247 2024-11-20T12:15:46,422 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting dc480c5f91df4e609c43a9b286d73981, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:46,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742147_1323 (size=13187) 2024-11-20T12:15:46,442 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:46,443 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/e74aa5388f0545e8945e471e0f05da7c is 50, key is test_row_0/C:col10/1732104945409/Put/seqid=0 2024-11-20T12:15:46,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742148_1324 (size=13187) 2024-11-20T12:15:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:46,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:46,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:46,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1015143611d74e55bd3da9673d1c7465 is 50, key is test_row_0/A:col10/1732104946719/Put/seqid=0 2024-11-20T12:15:46,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742149_1325 (size=12301) 2024-11-20T12:15:46,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105006728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105006729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105006732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105006732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,835 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/120f9cb0575b4d6c8382962ce2702189 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/120f9cb0575b4d6c8382962ce2702189 2024-11-20T12:15:46,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105006837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,839 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into 120f9cb0575b4d6c8382962ce2702189(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:46,839 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:46,839 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=13, startTime=1732104946375; duration=0sec 2024-11-20T12:15:46,839 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:46,839 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105006837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105006846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105006846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:46,860 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/e74aa5388f0545e8945e471e0f05da7c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e74aa5388f0545e8945e471e0f05da7c 2024-11-20T12:15:46,864 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into e74aa5388f0545e8945e471e0f05da7c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:46,864 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:46,865 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=13, startTime=1732104946375; duration=0sec 2024-11-20T12:15:46,865 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:46,865 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:47,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105007046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105007046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105007056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105007056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1015143611d74e55bd3da9673d1c7465 2024-11-20T12:15:47,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/7418e780d8594a93b5f4933ccc8a7f3d is 50, key is test_row_0/B:col10/1732104946719/Put/seqid=0 2024-11-20T12:15:47,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742150_1326 (size=12301) 2024-11-20T12:15:47,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105007357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105007357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105007367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105007367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-20T12:15:47,417 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:47,417 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-20T12:15:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T12:15:47,419 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:47,419 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:47,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57242 deadline: 1732105007437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,438 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in ceaa0a251a1510d6c7c95683bd6968dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T12:15:47,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/7418e780d8594a93b5f4933ccc8a7f3d 2024-11-20T12:15:47,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4f7a31d92b214dc09c39b831f8f3ea1a is 50, key is test_row_0/C:col10/1732104946719/Put/seqid=0 2024-11-20T12:15:47,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742151_1327 (size=12301) 2024-11-20T12:15:47,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T12:15:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T12:15:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105007866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105007866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T12:15:47,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:47,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105007876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105007876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:47,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:47,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4f7a31d92b214dc09c39b831f8f3ea1a 2024-11-20T12:15:47,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1015143611d74e55bd3da9673d1c7465 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465 2024-11-20T12:15:47,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T12:15:47,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/7418e780d8594a93b5f4933ccc8a7f3d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d 2024-11-20T12:15:47,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T12:15:47,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4f7a31d92b214dc09c39b831f8f3ea1a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a 2024-11-20T12:15:47,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T12:15:47,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ceaa0a251a1510d6c7c95683bd6968dd in 1257ms, sequenceid=413, compaction requested=false 2024-11-20T12:15:47,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:48,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-20T12:15:48,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,031 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:48,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1af1e8adc04d4591aa53334a23ef6ac0 is 50, key is test_row_0/A:col10/1732104946731/Put/seqid=0 2024-11-20T12:15:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T12:15:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742152_1328 (size=12301) 2024-11-20T12:15:48,051 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1af1e8adc04d4591aa53334a23ef6ac0 2024-11-20T12:15:48,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c451b25484ca4e94b15527a5f91ef9b1 is 50, key is test_row_0/B:col10/1732104946731/Put/seqid=0 2024-11-20T12:15:48,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742153_1329 (size=12301) 2024-11-20T12:15:48,490 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c451b25484ca4e94b15527a5f91ef9b1 2024-11-20T12:15:48,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3a43e412fd9940438f0475b06da183b3 is 50, key is test_row_0/C:col10/1732104946731/Put/seqid=0 2024-11-20T12:15:48,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742154_1330 (size=12301) 2024-11-20T12:15:48,501 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3a43e412fd9940438f0475b06da183b3 2024-11-20T12:15:48,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/1af1e8adc04d4591aa53334a23ef6ac0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0 2024-11-20T12:15:48,510 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T12:15:48,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c451b25484ca4e94b15527a5f91ef9b1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1 2024-11-20T12:15:48,515 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T12:15:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/3a43e412fd9940438f0475b06da183b3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3 2024-11-20T12:15:48,520 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T12:15:48,520 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for ceaa0a251a1510d6c7c95683bd6968dd in 490ms, sequenceid=427, compaction requested=true 2024-11-20T12:15:48,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-20T12:15:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-20T12:15:48,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T12:15:48,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1030 sec 2024-11-20T12:15:48,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.1070 sec 2024-11-20T12:15:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T12:15:48,556 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:48,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-20T12:15:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T12:15:48,558 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:48,559 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:48,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T12:15:48,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-20T12:15:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-20T12:15:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-20T12:15:48,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T12:15:48,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-20T12:15:48,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 158 msec 2024-11-20T12:15:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-20T12:15:48,878 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:48,878 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-20T12:15:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T12:15:48,881 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:48,882 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:48,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:48,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:48,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:48,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:48,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/48ed016114d64c0a8d7805c79ed779e1 is 50, key is test_row_0/A:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:48,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742155_1331 (size=14741) 2024-11-20T12:15:48,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/48ed016114d64c0a8d7805c79ed779e1 2024-11-20T12:15:48,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c56c361a6f1d431aa05056ec1d04519f is 50, key is test_row_0/B:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:48,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104948908, max timestamp=1732104948925Num uniques 859; 2024-11-20T12:15:48,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104948908, max timestamp=1732104948925Num uniques 859; 2024-11-20T12:15:48,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:48,928 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:48,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732104948908, max timestamp=1732104948925Num uniques 859; 2024-11-20T12:15:48,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T12:15:48,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742156_1332 (size=12301) 2024-11-20T12:15:48,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c56c361a6f1d431aa05056ec1d04519f 2024-11-20T12:15:48,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4749d0da41d8407f8a8486103c43ab2f is 50, key is test_row_0/C:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:48,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742157_1333 (size=12301) 2024-11-20T12:15:48,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4749d0da41d8407f8a8486103c43ab2f 2024-11-20T12:15:48,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/48ed016114d64c0a8d7805c79ed779e1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1 2024-11-20T12:15:48,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:48,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105008956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:48,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1, entries=200, sequenceid=438, filesize=14.4 K 2024-11-20T12:15:48,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/c56c361a6f1d431aa05056ec1d04519f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f 2024-11-20T12:15:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105008964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105008964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105008965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:48,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T12:15:48,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/4749d0da41d8407f8a8486103c43ab2f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f 2024-11-20T12:15:48,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T12:15:48,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ceaa0a251a1510d6c7c95683bd6968dd in 91ms, sequenceid=438, compaction requested=true 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:48,978 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:48,978 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ceaa0a251a1510d6c7c95683bd6968dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:48,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/A is initiating minor compaction (all files) 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/B is initiating minor compaction (all files) 2024-11-20T12:15:48,979 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/A in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,979 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/B in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:48,979 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/120f9cb0575b4d6c8382962ce2702189, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=51.3 K 2024-11-20T12:15:48,979 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/02838794ca424c31a129c354025cc035, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=48.9 K 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120f9cb0575b4d6c8382962ce2702189, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:48,979 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 02838794ca424c31a129c354025cc035, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1015143611d74e55bd3da9673d1c7465, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732104945571 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7418e780d8594a93b5f4933ccc8a7f3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732104945571 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1af1e8adc04d4591aa53334a23ef6ac0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104946724 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c451b25484ca4e94b15527a5f91ef9b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104946724 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ed016114d64c0a8d7805c79ed779e1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732104948881 2024-11-20T12:15:48,980 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c56c361a6f1d431aa05056ec1d04519f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732104948881 2024-11-20T12:15:48,989 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#B#compaction#288 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:48,989 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/64d0829b5362482594b1823eede24e6f is 50, key is test_row_0/B:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T12:15:48,990 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#A#compaction#287 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:48,991 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/adbcfce04a2949a58f4a77125e5a6941 is 50, key is test_row_0/A:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:48,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742159_1335 (size=13323) 2024-11-20T12:15:48,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742158_1334 (size=13323) 2024-11-20T12:15:49,010 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/adbcfce04a2949a58f4a77125e5a6941 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/adbcfce04a2949a58f4a77125e5a6941 2024-11-20T12:15:49,016 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/A of ceaa0a251a1510d6c7c95683bd6968dd into adbcfce04a2949a58f4a77125e5a6941(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:49,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:49,016 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/A, priority=12, startTime=1732104948978; duration=0sec 2024-11-20T12:15:49,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:49,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:A 2024-11-20T12:15:49,016 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:15:49,017 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/64d0829b5362482594b1823eede24e6f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/64d0829b5362482594b1823eede24e6f 2024-11-20T12:15:49,020 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:15:49,020 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ceaa0a251a1510d6c7c95683bd6968dd/C is initiating minor compaction (all files) 2024-11-20T12:15:49,020 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ceaa0a251a1510d6c7c95683bd6968dd/C in TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:49,020 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e74aa5388f0545e8945e471e0f05da7c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp, totalSize=48.9 K 2024-11-20T12:15:49,021 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74aa5388f0545e8945e471e0f05da7c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1732104945406 2024-11-20T12:15:49,021 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f7a31d92b214dc09c39b831f8f3ea1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732104945571 2024-11-20T12:15:49,025 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/B of ceaa0a251a1510d6c7c95683bd6968dd into 64d0829b5362482594b1823eede24e6f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:49,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:49,025 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/B, priority=12, startTime=1732104948978; duration=0sec 2024-11-20T12:15:49,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:49,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:B 2024-11-20T12:15:49,026 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a43e412fd9940438f0475b06da183b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732104946724 2024-11-20T12:15:49,026 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4749d0da41d8407f8a8486103c43ab2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732104948881 2024-11-20T12:15:49,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:49,035 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:49,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:15:49,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7c5d9534ee34f0bbf285c75cbbb61f1 is 50, key is test_row_0/A:col10/1732104948962/Put/seqid=0 2024-11-20T12:15:49,048 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ceaa0a251a1510d6c7c95683bd6968dd#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:49,049 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/adabc89184f549ce8c5b20ad79ee8c37 is 50, key is test_row_0/C:col10/1732104948886/Put/seqid=0 2024-11-20T12:15:49,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742160_1336 (size=12301) 2024-11-20T12:15:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:49,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:49,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105009085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742161_1337 (size=13323) 2024-11-20T12:15:49,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105009089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105009091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105009091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,112 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/adabc89184f549ce8c5b20ad79ee8c37 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/adabc89184f549ce8c5b20ad79ee8c37 2024-11-20T12:15:49,118 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ceaa0a251a1510d6c7c95683bd6968dd/C of ceaa0a251a1510d6c7c95683bd6968dd into adabc89184f549ce8c5b20ad79ee8c37(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:49,118 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:49,118 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., storeName=ceaa0a251a1510d6c7c95683bd6968dd/C, priority=12, startTime=1732104948978; duration=0sec 2024-11-20T12:15:49,119 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:49,119 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ceaa0a251a1510d6c7c95683bd6968dd:C 2024-11-20T12:15:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T12:15:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105009196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105009207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105009207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105009207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105009406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105009417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105009417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105009417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,466 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7c5d9534ee34f0bbf285c75cbbb61f1 2024-11-20T12:15:49,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/48fdb58fbfe34f288b4c3641903536ed is 50, key is test_row_0/B:col10/1732104948962/Put/seqid=0 2024-11-20T12:15:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742162_1338 (size=12301) 2024-11-20T12:15:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T12:15:49,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57208 deadline: 1732105009716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57222 deadline: 1732105009727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57192 deadline: 1732105009727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57244 deadline: 1732105009727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:15:49,878 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/48fdb58fbfe34f288b4c3641903536ed 2024-11-20T12:15:49,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/09166e1e7cc241d4836d03750c97466d is 50, key is test_row_0/C:col10/1732104948962/Put/seqid=0 2024-11-20T12:15:49,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742163_1339 (size=12301) 2024-11-20T12:15:49,888 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/09166e1e7cc241d4836d03750c97466d 2024-11-20T12:15:49,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/f7c5d9534ee34f0bbf285c75cbbb61f1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7c5d9534ee34f0bbf285c75cbbb61f1 2024-11-20T12:15:49,896 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7c5d9534ee34f0bbf285c75cbbb61f1, entries=150, sequenceid=477, filesize=12.0 K 2024-11-20T12:15:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/48fdb58fbfe34f288b4c3641903536ed as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/48fdb58fbfe34f288b4c3641903536ed 2024-11-20T12:15:49,901 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/48fdb58fbfe34f288b4c3641903536ed, entries=150, sequenceid=477, filesize=12.0 K 2024-11-20T12:15:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/09166e1e7cc241d4836d03750c97466d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/09166e1e7cc241d4836d03750c97466d 2024-11-20T12:15:49,905 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/09166e1e7cc241d4836d03750c97466d, entries=150, sequenceid=477, filesize=12.0 K 2024-11-20T12:15:49,906 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ceaa0a251a1510d6c7c95683bd6968dd in 871ms, sequenceid=477, compaction requested=false 2024-11-20T12:15:49,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:49,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:49,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-20T12:15:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-20T12:15:49,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-20T12:15:49,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-20T12:15:49,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.0310 sec 2024-11-20T12:15:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-20T12:15:50,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:50,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees 2024-11-20T12:15:50,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:50,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=92, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:50,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:50,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=93 2024-11-20T12:15:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:50,173 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:50,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8efcadf1d96948c8bacd33af465ea552 is 50, key is test_row_0/A:col10/1732104949088/Put/seqid=0 2024-11-20T12:15:50,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742164_1340 (size=12301) 2024-11-20T12:15:50,229 INFO [Thread-1148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1148. 2024-11-20T12:15:50,229 DEBUG [Thread-1148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,229 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,229 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-20T12:15:50,229 DEBUG [Thread-1148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,229 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,229 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,229 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,230 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-20T12:15:50,230 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,230 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,230 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,230 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. as already flushing 2024-11-20T12:15:50,231 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-20T12:15:50,231 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,231 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-20T12:15:50,231 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,231 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,231 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-20T12:15:50,231 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,231 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,237 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-20T12:15:50,237 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-20T12:15:50,237 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-20T12:15:50,237 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,237 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,237 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,237 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:50,237 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,238 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,238 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,238 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,238 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:50,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:50,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:50,582 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8efcadf1d96948c8bacd33af465ea552 2024-11-20T12:15:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/947b97aa2c0d44f2af31663ff7859a2c is 50, key is test_row_0/B:col10/1732104949088/Put/seqid=0 2024-11-20T12:15:50,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742165_1341 (size=12301) 2024-11-20T12:15:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:50,998 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/947b97aa2c0d44f2af31663ff7859a2c 2024-11-20T12:15:51,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a50e2cb91cf6477694d005c2172e57a8 is 50, key is test_row_0/C:col10/1732104949088/Put/seqid=0 2024-11-20T12:15:51,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742166_1342 (size=12301) 2024-11-20T12:15:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:51,410 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a50e2cb91cf6477694d005c2172e57a8 2024-11-20T12:15:51,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/8efcadf1d96948c8bacd33af465ea552 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8efcadf1d96948c8bacd33af465ea552 2024-11-20T12:15:51,423 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8efcadf1d96948c8bacd33af465ea552, entries=150, sequenceid=488, filesize=12.0 K 2024-11-20T12:15:51,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/947b97aa2c0d44f2af31663ff7859a2c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/947b97aa2c0d44f2af31663ff7859a2c 2024-11-20T12:15:51,429 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/947b97aa2c0d44f2af31663ff7859a2c, entries=150, sequenceid=488, filesize=12.0 K 2024-11-20T12:15:51,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/a50e2cb91cf6477694d005c2172e57a8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a50e2cb91cf6477694d005c2172e57a8 2024-11-20T12:15:51,434 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a50e2cb91cf6477694d005c2172e57a8, entries=150, sequenceid=488, filesize=12.0 K 2024-11-20T12:15:51,435 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for ceaa0a251a1510d6c7c95683bd6968dd in 1262ms, sequenceid=488, compaction requested=true 2024-11-20T12:15:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.HRegion(2603): Flush status journal for ceaa0a251a1510d6c7c95683bd6968dd: 2024-11-20T12:15:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:51,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=93}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=93 2024-11-20T12:15:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=93 2024-11-20T12:15:51,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-20T12:15:51,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-20T12:15:51,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=92, table=TestAcidGuarantees in 1.4200 sec 2024-11-20T12:15:51,468 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-20T12:15:51,468 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:15:51,468 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:51,468 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:51,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-20T12:15:52,167 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3234 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3125 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3241 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3233 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3125 2024-11-20T12:15:52,167 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:15:52,167 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:15:52,167 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:15:52,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:52,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:52,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:15:52,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:15:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T12:15:52,174 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104952173"}]},"ts":"1732104952173"} 2024-11-20T12:15:52,176 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:15:52,176 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:15:52,177 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:15:52,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, UNASSIGN}] 2024-11-20T12:15:52,180 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, UNASSIGN 2024-11-20T12:15:52,181 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ceaa0a251a1510d6c7c95683bd6968dd, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:52,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, UNASSIGN because future has completed 2024-11-20T12:15:52,183 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:15:52,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T12:15:52,336 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:52,336 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:15:52,336 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing ceaa0a251a1510d6c7c95683bd6968dd, disabling compactions & flushes 2024-11-20T12:15:52,336 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:52,336 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:52,336 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. after waiting 0 ms 2024-11-20T12:15:52,336 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:52,336 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(2902): Flushing ceaa0a251a1510d6c7c95683bd6968dd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=A 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=B 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ceaa0a251a1510d6c7c95683bd6968dd, store=C 2024-11-20T12:15:52,337 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:52,341 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7619fc137914ee19cfcae054db7e403 is 50, key is test_row_0/A:col10/1732104950236/Put/seqid=0 2024-11-20T12:15:52,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742167_1343 (size=12301) 2024-11-20T12:15:52,349 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7619fc137914ee19cfcae054db7e403 2024-11-20T12:15:52,359 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/9135b1fb36704b579ead9b76b23df746 is 50, key is test_row_0/B:col10/1732104950236/Put/seqid=0 2024-11-20T12:15:52,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742168_1344 (size=12301) 2024-11-20T12:15:52,389 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/9135b1fb36704b579ead9b76b23df746 2024-11-20T12:15:52,397 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/8bd8fefef807458db5a800ec2f72fff5 is 50, key is test_row_0/C:col10/1732104950236/Put/seqid=0 2024-11-20T12:15:52,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742169_1345 (size=12301) 2024-11-20T12:15:52,403 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/8bd8fefef807458db5a800ec2f72fff5 2024-11-20T12:15:52,409 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/A/d7619fc137914ee19cfcae054db7e403 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7619fc137914ee19cfcae054db7e403 2024-11-20T12:15:52,416 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7619fc137914ee19cfcae054db7e403, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T12:15:52,418 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/B/9135b1fb36704b579ead9b76b23df746 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/9135b1fb36704b579ead9b76b23df746 2024-11-20T12:15:52,424 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/9135b1fb36704b579ead9b76b23df746, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T12:15:52,425 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/.tmp/C/8bd8fefef807458db5a800ec2f72fff5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/8bd8fefef807458db5a800ec2f72fff5 2024-11-20T12:15:52,431 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/8bd8fefef807458db5a800ec2f72fff5, entries=150, sequenceid=496, filesize=12.0 K 2024-11-20T12:15:52,431 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ceaa0a251a1510d6c7c95683bd6968dd in 95ms, sequenceid=496, compaction requested=true 2024-11-20T12:15:52,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/5f683517aac04c1c9d8f3427d5863fee, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/55e582d416d549658a80a919fcc15b95, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8397489a4d004822803e91a56d150eb9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f9598f9b2bca4e1ab94586ca8ccbb26c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a3e0546cffc440c4bbfb257510fdf708, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/6c3118a1e89c421997cd278e67baf93c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/bc245862ee6945b38b34cf8a4782b6bb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/120f9cb0575b4d6c8382962ce2702189, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1] to archive 2024-11-20T12:15:52,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:52,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f97598a071484b65a6d5d29fb3a7b4d4 2024-11-20T12:15:52,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c74e4aa9f6484e8abb72ec68ca540532 2024-11-20T12:15:52,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/5f683517aac04c1c9d8f3427d5863fee to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/5f683517aac04c1c9d8f3427d5863fee 2024-11-20T12:15:52,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7d352e5be124fa6b1953427031e5d20 2024-11-20T12:15:52,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d065c6288cc74e3abdac0c5f9a5c1086 2024-11-20T12:15:52,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/c6e2f0d39d164f80becc25e74053593c 2024-11-20T12:15:52,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/55e582d416d549658a80a919fcc15b95 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/55e582d416d549658a80a919fcc15b95 2024-11-20T12:15:52,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a9c45d4137954c16a604fdb0a74da422 2024-11-20T12:15:52,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7fa506250e14fdab1becd3cdc48df13 2024-11-20T12:15:52,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8397489a4d004822803e91a56d150eb9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8397489a4d004822803e91a56d150eb9 2024-11-20T12:15:52,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7735d02477f4b89ae5818b16884bd0d 2024-11-20T12:15:52,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a7612d89180f42aa84f665c94473de66 2024-11-20T12:15:52,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/42d7475f0e1b4d5ca5c375a3f503460b 2024-11-20T12:15:52,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f9598f9b2bca4e1ab94586ca8ccbb26c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f9598f9b2bca4e1ab94586ca8ccbb26c 2024-11-20T12:15:52,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b085681c38f841219256dc730c7a69f6 2024-11-20T12:15:52,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/7571e32b9fb84742a32508f03f3dc90e 2024-11-20T12:15:52,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a3e0546cffc440c4bbfb257510fdf708 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a3e0546cffc440c4bbfb257510fdf708 2024-11-20T12:15:52,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/a4ab046e7a56487a9256580538d4b2b3 2024-11-20T12:15:52,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/6c3118a1e89c421997cd278e67baf93c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/6c3118a1e89c421997cd278e67baf93c 2024-11-20T12:15:52,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/b2e1aa53b36449839c91e37431edd477 2024-11-20T12:15:52,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/2a4b9802ec9f4b49b6a6e61b126257f9 2024-11-20T12:15:52,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/9e71b0ee654a44a6bbff5a6e86a1a1f6 2024-11-20T12:15:52,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8379ba3a285640d1963d6bffa0322798 2024-11-20T12:15:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T12:15:52,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/bc245862ee6945b38b34cf8a4782b6bb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/bc245862ee6945b38b34cf8a4782b6bb 2024-11-20T12:15:52,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/4e16e60ab2c14a6f91279a791e9cec8e 2024-11-20T12:15:52,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/280bda72a0e2432e8ba8b2c58ea5a9bc 2024-11-20T12:15:52,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/120f9cb0575b4d6c8382962ce2702189 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/120f9cb0575b4d6c8382962ce2702189 2024-11-20T12:15:52,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1015143611d74e55bd3da9673d1c7465 2024-11-20T12:15:52,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/1af1e8adc04d4591aa53334a23ef6ac0 2024-11-20T12:15:52,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/48ed016114d64c0a8d7805c79ed779e1 2024-11-20T12:15:52,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/25af8533860f4103b96ec7170ed7f335, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/bc987b797eb54142bbaaa0835696c77f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/73f5d9025e0e4ea7b628219db904f89d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/6065621f53134dd3aa7c04531d9b4a72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e4eb7b8fb44c42758a5bb268d1f2523d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3eaee48214524b2b8b683ec11eaf5f83, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3db3189b53cd47858c191935be5c360e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/02838794ca424c31a129c354025cc035, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f] to archive 2024-11-20T12:15:52,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:52,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/223b131a87a44f93b2233022b34efc0c 2024-11-20T12:15:52,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e32c471a99234a80a21a4e9ab562a33e 2024-11-20T12:15:52,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/25af8533860f4103b96ec7170ed7f335 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/25af8533860f4103b96ec7170ed7f335 2024-11-20T12:15:52,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/88771729cbd74122ada8edeadd0505f6 2024-11-20T12:15:52,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4624f55f315e47f8a4b58d0109128a82 2024-11-20T12:15:52,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/b5a600e55c0c4c06babeae0ff1190e90 2024-11-20T12:15:52,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/bc987b797eb54142bbaaa0835696c77f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/bc987b797eb54142bbaaa0835696c77f 2024-11-20T12:15:52,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/acd493a3a0984631be3cdcd5e52199d3 2024-11-20T12:15:52,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4b68d9f684be4121a0cf032eb67d7b12 2024-11-20T12:15:52,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/73f5d9025e0e4ea7b628219db904f89d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/73f5d9025e0e4ea7b628219db904f89d 2024-11-20T12:15:52,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14f4302630354a3bb93250518b77d210 2024-11-20T12:15:52,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/62548bdbf35b4f54929cc17753470dd6 2024-11-20T12:15:52,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/6065621f53134dd3aa7c04531d9b4a72 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/6065621f53134dd3aa7c04531d9b4a72 2024-11-20T12:15:52,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/dbea1b756cfe412c82ca304cfbc33dc8 2024-11-20T12:15:52,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e7870767aa7446cb85003f1a2809ec64 2024-11-20T12:15:52,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e4eb7b8fb44c42758a5bb268d1f2523d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/e4eb7b8fb44c42758a5bb268d1f2523d 2024-11-20T12:15:52,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/4917c56cfd6e48439cf25c4af5a159b2 2024-11-20T12:15:52,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/833b4e131eda4bc2b0ea3ea3443ab32a 2024-11-20T12:15:52,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3eaee48214524b2b8b683ec11eaf5f83 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3eaee48214524b2b8b683ec11eaf5f83 2024-11-20T12:15:52,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3c3a5257b6b9484982e9a0cb8cfa6c74 2024-11-20T12:15:52,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/10bb07ec8b864cbca369d7e89d9f147b 2024-11-20T12:15:52,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/14fab086c893464a846a6789f1714142 2024-11-20T12:15:52,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3db3189b53cd47858c191935be5c360e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/3db3189b53cd47858c191935be5c360e 2024-11-20T12:15:52,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/758c6d372151477487987838f5470dca 2024-11-20T12:15:52,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/8ccaa5fc8f034791a5bf859ea6810cd0 2024-11-20T12:15:52,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/02838794ca424c31a129c354025cc035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/02838794ca424c31a129c354025cc035 2024-11-20T12:15:52,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c6108f2971a7423c8486dbdc88a476da 2024-11-20T12:15:52,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/7418e780d8594a93b5f4933ccc8a7f3d 2024-11-20T12:15:52,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c451b25484ca4e94b15527a5f91ef9b1 2024-11-20T12:15:52,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/c56c361a6f1d431aa05056ec1d04519f 2024-11-20T12:15:52,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c1752abe147b41719a8e84dbdf68d075, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/005650713d3e49f09b6bb01f469a2868, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/872ea7d7720a474d95baef88884636a2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/1e7a5374dbb148fd9e1538efeec80c4c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6f1d15c5851c45f3b1c41cda3d067548, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e18fbe7dd2e14d0ba63bc611ac19cdf5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/b24655cdeb524f0d90cfab8a392fd1c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e74aa5388f0545e8945e471e0f05da7c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f] to archive 2024-11-20T12:15:52,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:15:52,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/00772f4b731b48058b5cf1e1043ea150 2024-11-20T12:15:52,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/86d98c035e9249c593c29627abe00fc0 2024-11-20T12:15:52,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c1752abe147b41719a8e84dbdf68d075 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c1752abe147b41719a8e84dbdf68d075 2024-11-20T12:15:52,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/ade35d9f0de6433b9408c238c3e53b6c 2024-11-20T12:15:52,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/f61f917510934dd5904f14dbbbe72f55 2024-11-20T12:15:52,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/d3246f3ed83d4de798b2a46ef65809f7 2024-11-20T12:15:52,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/005650713d3e49f09b6bb01f469a2868 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/005650713d3e49f09b6bb01f469a2868 2024-11-20T12:15:52,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3894e26ef6bf4a1ea89dfcc60b9f5e53 2024-11-20T12:15:52,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/17727838ccd74f97b93a0a146a9c2066 2024-11-20T12:15:52,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/872ea7d7720a474d95baef88884636a2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/872ea7d7720a474d95baef88884636a2 2024-11-20T12:15:52,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3c5f8031ca8b4bb181cb0452cd45f187 2024-11-20T12:15:52,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/98cc0e07987c4ed7b3bb4432dd6f63c6 2024-11-20T12:15:52,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/1e7a5374dbb148fd9e1538efeec80c4c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/1e7a5374dbb148fd9e1538efeec80c4c 2024-11-20T12:15:52,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4c050fc535d14eb1a70e52946641d6ba 2024-11-20T12:15:52,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/692beb2aa2124baba1568038d46e77e9 2024-11-20T12:15:52,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6f1d15c5851c45f3b1c41cda3d067548 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6f1d15c5851c45f3b1c41cda3d067548 2024-11-20T12:15:52,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/43e88fdd75e24123b52c3d696f671606 2024-11-20T12:15:52,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/c52991e7e4314b539a48e9b760408a89 2024-11-20T12:15:52,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e18fbe7dd2e14d0ba63bc611ac19cdf5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e18fbe7dd2e14d0ba63bc611ac19cdf5 2024-11-20T12:15:52,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/cc81aa456c3b45caa549d6efd669fced 2024-11-20T12:15:52,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a6cfb55f530f4ea79d8f9eb7cbfd4a9b 2024-11-20T12:15:52,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dbf4e4bac732414cbeb5a2a1c3349bed 2024-11-20T12:15:52,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/b24655cdeb524f0d90cfab8a392fd1c8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/b24655cdeb524f0d90cfab8a392fd1c8 2024-11-20T12:15:52,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/6d21832d8e2a487ea1d795f1134efd27 2024-11-20T12:15:52,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a4d9bbeba82c43f8b43a691781b60b4b 2024-11-20T12:15:52,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e74aa5388f0545e8945e471e0f05da7c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/e74aa5388f0545e8945e471e0f05da7c 2024-11-20T12:15:52,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/dc480c5f91df4e609c43a9b286d73981 2024-11-20T12:15:52,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4f7a31d92b214dc09c39b831f8f3ea1a 2024-11-20T12:15:52,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/3a43e412fd9940438f0475b06da183b3 2024-11-20T12:15:52,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/4749d0da41d8407f8a8486103c43ab2f 2024-11-20T12:15:52,653 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/recovered.edits/499.seqid, newMaxSeqId=499, maxSeqId=1 2024-11-20T12:15:52,654 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. 2024-11-20T12:15:52,655 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for ceaa0a251a1510d6c7c95683bd6968dd: Waiting for close lock at 1732104952336Running coprocessor pre-close hooks at 1732104952336Disabling compacts and flushes for region at 1732104952336Disabling writes for close at 1732104952336Obtaining lock to block concurrent updates at 1732104952336Preparing flush snapshotting stores in ceaa0a251a1510d6c7c95683bd6968dd at 1732104952336Finished memstore snapshotting TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732104952337 (+1 ms)Flushing stores of TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd. at 1732104952338 (+1 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/A: creating writer at 1732104952338Flushing ceaa0a251a1510d6c7c95683bd6968dd/A: appending metadata at 1732104952341 (+3 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/A: closing flushed file at 1732104952341Flushing ceaa0a251a1510d6c7c95683bd6968dd/B: creating writer at 1732104952355 (+14 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/B: appending metadata at 1732104952358 (+3 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/B: closing flushed file at 1732104952358Flushing ceaa0a251a1510d6c7c95683bd6968dd/C: creating writer at 1732104952393 (+35 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/C: appending metadata at 1732104952397 (+4 ms)Flushing ceaa0a251a1510d6c7c95683bd6968dd/C: closing flushed file at 1732104952397Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@429f986b: reopening flushed file at 1732104952408 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@448a4dcc: reopening flushed file at 1732104952417 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33378645: reopening flushed file at 1732104952424 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ceaa0a251a1510d6c7c95683bd6968dd in 95ms, sequenceid=496, compaction requested=true at 1732104952431 (+7 ms)Writing region close event to WAL at 1732104952649 (+218 ms)Running coprocessor post-close hooks at 1732104952654 (+5 ms)Closed at 1732104952654 2024-11-20T12:15:52,657 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:52,657 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=ceaa0a251a1510d6c7c95683bd6968dd, regionState=CLOSED 2024-11-20T12:15:52,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:52,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-20T12:15:52,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure ceaa0a251a1510d6c7c95683bd6968dd, server=843d83d51ba0,42825,1732104864746 in 477 msec 2024-11-20T12:15:52,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T12:15:52,664 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ceaa0a251a1510d6c7c95683bd6968dd, UNASSIGN in 483 msec 2024-11-20T12:15:52,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-20T12:15:52,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-20T12:15:52,669 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104952669"}]},"ts":"1732104952669"} 2024-11-20T12:15:52,673 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:15:52,673 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:15:52,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 505 msec 2024-11-20T12:15:52,704 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:15:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-20T12:15:52,797 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:52,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:15:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,800 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=98, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-20T12:15:52,801 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=98, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:52,807 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/recovered.edits] 2024-11-20T12:15:52,814 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8efcadf1d96948c8bacd33af465ea552 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/8efcadf1d96948c8bacd33af465ea552 2024-11-20T12:15:52,816 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/adbcfce04a2949a58f4a77125e5a6941 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/adbcfce04a2949a58f4a77125e5a6941 2024-11-20T12:15:52,823 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7619fc137914ee19cfcae054db7e403 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/d7619fc137914ee19cfcae054db7e403 2024-11-20T12:15:52,824 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7c5d9534ee34f0bbf285c75cbbb61f1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/A/f7c5d9534ee34f0bbf285c75cbbb61f1 2024-11-20T12:15:52,827 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/48fdb58fbfe34f288b4c3641903536ed to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/48fdb58fbfe34f288b4c3641903536ed 2024-11-20T12:15:52,832 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/64d0829b5362482594b1823eede24e6f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/64d0829b5362482594b1823eede24e6f 2024-11-20T12:15:52,834 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/9135b1fb36704b579ead9b76b23df746 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/9135b1fb36704b579ead9b76b23df746 2024-11-20T12:15:52,837 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/947b97aa2c0d44f2af31663ff7859a2c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/B/947b97aa2c0d44f2af31663ff7859a2c 2024-11-20T12:15:52,839 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/09166e1e7cc241d4836d03750c97466d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/09166e1e7cc241d4836d03750c97466d 2024-11-20T12:15:52,840 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/8bd8fefef807458db5a800ec2f72fff5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/8bd8fefef807458db5a800ec2f72fff5 2024-11-20T12:15:52,841 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a50e2cb91cf6477694d005c2172e57a8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/a50e2cb91cf6477694d005c2172e57a8 2024-11-20T12:15:52,842 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/adabc89184f549ce8c5b20ad79ee8c37 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/C/adabc89184f549ce8c5b20ad79ee8c37 2024-11-20T12:15:52,848 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/recovered.edits/499.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd/recovered.edits/499.seqid 2024-11-20T12:15:52,849 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ceaa0a251a1510d6c7c95683bd6968dd 2024-11-20T12:15:52,849 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:15:52,852 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=98, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,856 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:15:52,860 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:15:52,861 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=98, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,861 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:15:52,861 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732104952861"}]},"ts":"9223372036854775807"} 2024-11-20T12:15:52,864 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:15:52,864 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ceaa0a251a1510d6c7c95683bd6968dd, NAME => 'TestAcidGuarantees,,1732104927994.ceaa0a251a1510d6c7c95683bd6968dd.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:15:52,864 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:15:52,864 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732104952864"}]},"ts":"9223372036854775807"} 2024-11-20T12:15:52,867 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:15:52,868 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=98, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-20T12:15:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-20T12:15:52,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:15:52,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:52,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 250), OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=505 (was 465) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3219 (was 2668) - AvailableMemoryMB LEAK? - 2024-11-20T12:15:52,930 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=505, ProcessCount=11, AvailableMemoryMB=3217 2024-11-20T12:15:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:15:52,931 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:15:52,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:52,934 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:15:52,934 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:52,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 99 2024-11-20T12:15:52,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T12:15:52,935 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:15:52,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742170_1346 (size=1061) 2024-11-20T12:15:52,949 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:15:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742171_1347 (size=53) 2024-11-20T12:15:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T12:15:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T12:15:53,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:53,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bdb95162a28369d7de140a1abe1b7035, disabling compactions & flushes 2024-11-20T12:15:53,362 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. after waiting 0 ms 2024-11-20T12:15:53,362 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,362 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,363 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bdb95162a28369d7de140a1abe1b7035: Waiting for close lock at 1732104953362Disabling compacts and flushes for region at 1732104953362Disabling writes for close at 1732104953362Writing region close event to WAL at 1732104953362Closed at 1732104953362 2024-11-20T12:15:53,364 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:15:53,364 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732104953364"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104953364"}]},"ts":"1732104953364"} 2024-11-20T12:15:53,365 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:15:53,366 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:15:53,367 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104953366"}]},"ts":"1732104953366"} 2024-11-20T12:15:53,368 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:15:53,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, ASSIGN}] 2024-11-20T12:15:53,369 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, ASSIGN 2024-11-20T12:15:53,370 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:15:53,520 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:53,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, ASSIGN because future has completed 2024-11-20T12:15:53,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T12:15:53,677 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,677 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7752): Opening region: {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:15:53,678 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,678 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:53,678 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7794): checking encryption for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,678 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7797): checking classloading for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,680 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,682 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:53,682 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName A 2024-11-20T12:15:53,682 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:53,683 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:53,683 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,684 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:53,684 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName B 2024-11-20T12:15:53,684 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:53,685 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:53,685 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,686 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:53,686 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName C 2024-11-20T12:15:53,686 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:53,686 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:53,686 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,686 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1038): replaying wal for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,687 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,687 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,688 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1048): stopping wal replay for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,688 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1060): Cleaning up temporary data for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,689 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:15:53,689 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1093): writing seq id for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,691 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:15:53,691 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1114): Opened bdb95162a28369d7de140a1abe1b7035; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60539492, jitterRate=-0.09789127111434937}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:15:53,691 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:53,692 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1006): Region open journal for bdb95162a28369d7de140a1abe1b7035: Running coprocessor pre-open hook at 1732104953678Writing region info on filesystem at 1732104953678Initializing all the Stores at 1732104953679 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104953679Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104953679Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104953680 (+1 ms)Cleaning up temporary data from old regions at 1732104953688 (+8 ms)Running coprocessor post-open hooks at 1732104953691 (+3 ms)Region opened successfully at 1732104953692 (+1 ms) 2024-11-20T12:15:53,692 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., pid=101, masterSystemTime=1732104953674 2024-11-20T12:15:53,694 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,694 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:53,694 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:53,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:53,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-20T12:15:53,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 in 174 msec 2024-11-20T12:15:53,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T12:15:53,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, ASSIGN in 329 msec 2024-11-20T12:15:53,699 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:15:53,699 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104953699"}]},"ts":"1732104953699"} 2024-11-20T12:15:53,701 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:15:53,701 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=99, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:15:53,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 770 msec 2024-11-20T12:15:54,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T12:15:54,067 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:54,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48dcb24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:54,068 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:54,068 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:54,069 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15448242, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:54,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:54,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:54,072 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:54,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d3b3726, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:54,073 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:54,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:54,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:54,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:54,075 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:15:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:15:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4f168542 2024-11-20T12:15:54,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:15:54,077 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:15:54,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:15:54,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:15:54,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T12:15:54,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742172_1348 (size=1097) 2024-11-20T12:15:54,495 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T12:15:54,496 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T12:15:54,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:15:54,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE}] 2024-11-20T12:15:54,500 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE 2024-11-20T12:15:54,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:54,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE because future has completed 2024-11-20T12:15:54,502 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:15:54,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:54,654 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(122): Close bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:15:54,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1722): Closing bdb95162a28369d7de140a1abe1b7035, disabling compactions & flushes 2024-11-20T12:15:54,654 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. after waiting 0 ms 2024-11-20T12:15:54,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,661 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:15:54,662 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,662 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegion(1676): Region close journal for bdb95162a28369d7de140a1abe1b7035: Waiting for close lock at 1732104954654Running coprocessor pre-close hooks at 1732104954654Disabling compacts and flushes for region at 1732104954654Disabling writes for close at 1732104954654Writing region close event to WAL at 1732104954655 (+1 ms)Running coprocessor post-close hooks at 1732104954662 (+7 ms)Closed at 1732104954662 2024-11-20T12:15:54,662 WARN [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] regionserver.HRegionServer(3299): Not adding moved region record: bdb95162a28369d7de140a1abe1b7035 to self. 2024-11-20T12:15:54,664 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=105}] handler.UnassignRegionHandler(157): Closed bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=CLOSED 2024-11-20T12:15:54,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-20T12:15:54,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 in 165 msec 2024-11-20T12:15:54,670 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE; state=CLOSED, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=true 2024-11-20T12:15:54,820 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:54,822 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE because future has completed 2024-11-20T12:15:54,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:15:54,977 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:15:54,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:15:54,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,979 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,980 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:54,981 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName A 2024-11-20T12:15:54,982 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:54,982 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:54,983 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,984 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:54,984 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName B 2024-11-20T12:15:54,984 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:54,984 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:54,984 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,985 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:15:54,985 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bdb95162a28369d7de140a1abe1b7035 columnFamilyName C 2024-11-20T12:15:54,985 DEBUG [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:54,986 INFO [StoreOpener-bdb95162a28369d7de140a1abe1b7035-1 {}] regionserver.HStore(327): Store=bdb95162a28369d7de140a1abe1b7035/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:15:54,986 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,986 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,987 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,988 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,990 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,990 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,990 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:15:54,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,994 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened bdb95162a28369d7de140a1abe1b7035; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59959319, jitterRate=-0.10653652250766754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:15:54,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:54,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for bdb95162a28369d7de140a1abe1b7035: Running coprocessor pre-open hook at 1732104954978Writing region info on filesystem at 1732104954978Initializing all the Stores at 1732104954978Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104954978Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104954978Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104954979 (+1 ms)Cleaning up temporary data from old regions at 1732104954990 (+11 ms)Running coprocessor post-open hooks at 1732104954994 (+4 ms)Region opened successfully at 1732104954994 2024-11-20T12:15:54,995 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., pid=106, masterSystemTime=1732104954974 2024-11-20T12:15:54,998 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,998 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:54,999 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=104 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=OPEN, openSeqNum=5, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=104, state=RUNNABLE, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:15:55,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=104 2024-11-20T12:15:55,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=104, state=SUCCESS, hasLock=false; OpenRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 in 180 msec 2024-11-20T12:15:55,004 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40015 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,42825,1732104864746, table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:15:55,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T12:15:55,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, REOPEN/MOVE in 505 msec 2024-11-20T12:15:55,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-20T12:15:55,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-20T12:15:55,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 930 msec 2024-11-20T12:15:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-11-20T12:15:55,013 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:55,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,014 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8735c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,015 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,015 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28234a11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,017 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6226be6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a65e4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,019 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,019 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,020 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76eeeaa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,021 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c41677f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,022 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@334d3f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72937f8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32da4b0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,025 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab2e602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b17aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370f22ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c5d5dc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a210db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,033 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111dddf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,036 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37fff448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,038 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca890dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f63200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,041 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f748db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf0e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:15:55,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:15:55,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:15:55,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:15:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:15:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13f89be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:15:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:15:55,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:15:55,044 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:15:55,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f116a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:15:55,044 DEBUG [Thread-1583 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,044 DEBUG [Thread-1579 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,045 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,045 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,045 DEBUG [Thread-1592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,045 DEBUG [Thread-1577 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,045 DEBUG [Thread-1585 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,046 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T12:15:55,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,047 DEBUG [Thread-1577 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,047 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,048 DEBUG [Thread-1577 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,048 DEBUG [Thread-1581 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,049 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:15:55,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:15:55,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:55,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:15:55,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,053 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:15:55,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,059 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:15:55,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:55,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:15:55,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:55,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be5251c86ed44b5aab46558360eff56c_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104955060/Put/seqid=0 2024-11-20T12:15:55,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105015105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742173_1349 (size=17034) 2024-11-20T12:15:55,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105015109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105015111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105015112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105015113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:55,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105015216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,225 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105015226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105015226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105015237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105015237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105015426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105015447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,583 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:55,588 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120be5251c86ed44b5aab46558360eff56c_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5251c86ed44b5aab46558360eff56c_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:55,589 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cbde4c5bde8c41508216c8ef520ea185, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:55,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cbde4c5bde8c41508216c8ef520ea185 is 175, key is test_row_0/A:col10/1732104955060/Put/seqid=0 2024-11-20T12:15:55,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742174_1350 (size=48139) 2024-11-20T12:15:55,598 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cbde4c5bde8c41508216c8ef520ea185 2024-11-20T12:15:55,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/bca79b8584eb449690febd5e9a7e1a7e is 50, key is test_row_0/B:col10/1732104955060/Put/seqid=0 2024-11-20T12:15:55,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742175_1351 (size=12001) 2024-11-20T12:15:55,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/bca79b8584eb449690febd5e9a7e1a7e 2024-11-20T12:15:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:55,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/d14db7f4a2e9464080515d9f4d9f0a6d is 50, key is test_row_0/C:col10/1732104955060/Put/seqid=0 2024-11-20T12:15:55,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742176_1352 (size=12001) 2024-11-20T12:15:55,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:55,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105015747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105015757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105015767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105015767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105015767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:55,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:55,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:55,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:55,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:55,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:56,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:56,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:56,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:56,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/d14db7f4a2e9464080515d9f4d9f0a6d 2024-11-20T12:15:56,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cbde4c5bde8c41508216c8ef520ea185 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185 2024-11-20T12:15:56,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185, entries=250, sequenceid=16, filesize=47.0 K 2024-11-20T12:15:56,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/bca79b8584eb449690febd5e9a7e1a7e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e 2024-11-20T12:15:56,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T12:15:56,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/d14db7f4a2e9464080515d9f4d9f0a6d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d 2024-11-20T12:15:56,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T12:15:56,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bdb95162a28369d7de140a1abe1b7035 in 1077ms, sequenceid=16, compaction requested=false 2024-11-20T12:15:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:56,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:56,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:56,194 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:15:56,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d5c3e08fdee64db79dc5707722afca36_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104955109/Put/seqid=0 2024-11-20T12:15:56,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742177_1353 (size=12154) 2024-11-20T12:15:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:56,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105016287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105016287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105016292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105016296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105016297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105016408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105016408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105016408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105016417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:56,613 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d5c3e08fdee64db79dc5707722afca36_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d5c3e08fdee64db79dc5707722afca36_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/766f56e9de3d4538bb0a3b111cd24819, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:56,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/766f56e9de3d4538bb0a3b111cd24819 is 175, key is test_row_0/A:col10/1732104955109/Put/seqid=0 2024-11-20T12:15:56,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105016619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105016619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105016619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742178_1354 (size=30955) 2024-11-20T12:15:56,649 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/766f56e9de3d4538bb0a3b111cd24819 2024-11-20T12:15:56,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105016636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/8b915b44dc5b4d668d0a3c1daa02bded is 50, key is test_row_0/B:col10/1732104955109/Put/seqid=0 2024-11-20T12:15:56,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742179_1355 (size=12001) 2024-11-20T12:15:56,721 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/8b915b44dc5b4d668d0a3c1daa02bded 2024-11-20T12:15:56,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/543ca0381de741e594e75544d31c0d19 is 50, key is test_row_0/C:col10/1732104955109/Put/seqid=0 2024-11-20T12:15:56,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742180_1356 (size=12001) 2024-11-20T12:15:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105016937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105016938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105016940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:56,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:56,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105016966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:56,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,157 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/543ca0381de741e594e75544d31c0d19 2024-11-20T12:15:57,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/766f56e9de3d4538bb0a3b111cd24819 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819 2024-11-20T12:15:57,168 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819, entries=150, sequenceid=36, filesize=30.2 K 2024-11-20T12:15:57,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/8b915b44dc5b4d668d0a3c1daa02bded as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded 2024-11-20T12:15:57,174 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded, entries=150, sequenceid=36, filesize=11.7 K 2024-11-20T12:15:57,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/543ca0381de741e594e75544d31c0d19 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19 2024-11-20T12:15:57,196 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19, entries=150, sequenceid=36, filesize=11.7 K 2024-11-20T12:15:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:57,197 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bdb95162a28369d7de140a1abe1b7035 in 1003ms, sequenceid=36, compaction requested=false 2024-11-20T12:15:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T12:15:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T12:15:57,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T12:15:57,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1490 sec 2024-11-20T12:15:57,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1570 sec 2024-11-20T12:15:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:57,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:15:57,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:15:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:15:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:15:57,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:57,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202ec22aa1660c44b59becc79b5a063562_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:57,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742181_1357 (size=12154) 2024-11-20T12:15:57,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:57,431 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202ec22aa1660c44b59becc79b5a063562_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202ec22aa1660c44b59becc79b5a063562_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:57,432 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/67097fca0d5f4db6baf92421eaeba9bb, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:57,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/67097fca0d5f4db6baf92421eaeba9bb is 175, key is test_row_0/A:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:57,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742182_1358 (size=30955) 2024-11-20T12:15:57,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105017458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105017461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105017464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105017473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105017486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105017587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105017587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105017597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105017807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105017807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:57,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105017816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:57,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:57,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/67097fca0d5f4db6baf92421eaeba9bb 2024-11-20T12:15:57,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5d872020c1bb4de0a80530265d351f9f is 50, key is test_row_0/B:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:57,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742183_1359 (size=12001) 2024-11-20T12:15:57,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5d872020c1bb4de0a80530265d351f9f 2024-11-20T12:15:57,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/43b95fb95d334112b2b4ced8b1ce3a9f is 50, key is test_row_0/C:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:57,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742184_1360 (size=12001) 2024-11-20T12:15:57,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/43b95fb95d334112b2b4ced8b1ce3a9f 2024-11-20T12:15:57,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/67097fca0d5f4db6baf92421eaeba9bb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb 2024-11-20T12:15:57,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb, entries=150, sequenceid=49, filesize=30.2 K 2024-11-20T12:15:57,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5d872020c1bb4de0a80530265d351f9f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f 2024-11-20T12:15:57,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:15:57,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/43b95fb95d334112b2b4ced8b1ce3a9f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f 2024-11-20T12:15:57,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:15:57,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for bdb95162a28369d7de140a1abe1b7035 in 674ms, sequenceid=49, compaction requested=true 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:57,996 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:57,996 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:15:57,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:58,000 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:58,000 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:15:58,000 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:58,000 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:58,000 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:15:58,000 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:58,000 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=107.5 K 2024-11-20T12:15:58,000 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:58,000 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb] 2024-11-20T12:15:58,000 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=35.2 K 2024-11-20T12:15:58,003 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cbde4c5bde8c41508216c8ef520ea185, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732104955056 2024-11-20T12:15:58,003 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bca79b8584eb449690febd5e9a7e1a7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732104955060 2024-11-20T12:15:58,003 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b915b44dc5b4d668d0a3c1daa02bded, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732104955087 2024-11-20T12:15:58,003 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 766f56e9de3d4538bb0a3b111cd24819, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732104955087 2024-11-20T12:15:58,004 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d872020c1bb4de0a80530265d351f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:15:58,004 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 67097fca0d5f4db6baf92421eaeba9bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:15:58,016 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:58,030 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e1cfb73b3cd74a379385d65d81d229ce_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:58,032 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:58,032 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/116280921b3045aea77f81bfe3438aea is 50, key is test_row_0/B:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:58,033 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e1cfb73b3cd74a379385d65d81d229ce_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:58,033 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e1cfb73b3cd74a379385d65d81d229ce_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:58,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742185_1361 (size=12104) 2024-11-20T12:15:58,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742186_1362 (size=4469) 2024-11-20T12:15:58,050 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#308 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:58,050 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/585775ee23c647c7a093efacca43e2d4 is 175, key is test_row_0/A:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:58,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742187_1363 (size=31058) 2024-11-20T12:15:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:58,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:15:58,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:58,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120652f0941e5bc4f70a44337302bec4369_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104958131/Put/seqid=0 2024-11-20T12:15:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742188_1364 (size=14594) 2024-11-20T12:15:58,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:58,147 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120652f0941e5bc4f70a44337302bec4369_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120652f0941e5bc4f70a44337302bec4369_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:58,147 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0e76139abcc34e3aa3b2184233f97c18, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:58,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0e76139abcc34e3aa3b2184233f97c18 is 175, key is test_row_0/A:col10/1732104958131/Put/seqid=0 2024-11-20T12:15:58,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742189_1365 (size=39549) 2024-11-20T12:15:58,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105018166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105018167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105018172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,187 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:15:58,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105018277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105018287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105018287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,447 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/116280921b3045aea77f81bfe3438aea as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/116280921b3045aea77f81bfe3438aea 2024-11-20T12:15:58,451 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into 116280921b3045aea77f81bfe3438aea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:58,451 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:58,451 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=13, startTime=1732104957996; duration=0sec 2024-11-20T12:15:58,451 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:15:58,451 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:15:58,451 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:15:58,452 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:15:58,452 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:15:58,452 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:58,452 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=35.2 K 2024-11-20T12:15:58,453 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d14db7f4a2e9464080515d9f4d9f0a6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732104955060 2024-11-20T12:15:58,454 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 543ca0381de741e594e75544d31c0d19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732104955087 2024-11-20T12:15:58,455 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b95fb95d334112b2b4ced8b1ce3a9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:15:58,459 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/585775ee23c647c7a093efacca43e2d4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4 2024-11-20T12:15:58,462 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:15:58,463 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into 585775ee23c647c7a093efacca43e2d4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:58,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:58,463 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=13, startTime=1732104957996; duration=0sec 2024-11-20T12:15:58,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:58,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:15:58,463 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/dd1353769cef4598b18e23aec2325cb4 is 50, key is test_row_0/C:col10/1732104956291/Put/seqid=0 2024-11-20T12:15:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742190_1366 (size=12104) 2024-11-20T12:15:58,470 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/dd1353769cef4598b18e23aec2325cb4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/dd1353769cef4598b18e23aec2325cb4 2024-11-20T12:15:58,474 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into dd1353769cef4598b18e23aec2325cb4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:15:58,474 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:58,474 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=13, startTime=1732104957996; duration=0sec 2024-11-20T12:15:58,474 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:15:58,474 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:15:58,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105018487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,494 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105018497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105018497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105018506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105018506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,512 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0e76139abcc34e3aa3b2184233f97c18 2024-11-20T12:15:58,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/407292de193944058be7ed62805e9698 is 50, key is test_row_0/B:col10/1732104958131/Put/seqid=0 2024-11-20T12:15:58,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742191_1367 (size=12001) 2024-11-20T12:15:58,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105018817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105018817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105018817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:58,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:58,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/407292de193944058be7ed62805e9698 2024-11-20T12:15:58,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/3bd133f7ed1f4131bd54966ad48c8353 is 50, key is test_row_0/C:col10/1732104958131/Put/seqid=0 2024-11-20T12:15:58,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742192_1368 (size=12001) 2024-11-20T12:15:59,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T12:15:59,206 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:15:59,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T12:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T12:15:59,208 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:15:59,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:15:59,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T12:15:59,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105019336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:59,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105019336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:59,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105019336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:15:59,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:15:59,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T12:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:15:59,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/3bd133f7ed1f4131bd54966ad48c8353 2024-11-20T12:15:59,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0e76139abcc34e3aa3b2184233f97c18 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18 2024-11-20T12:15:59,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18, entries=200, sequenceid=69, filesize=38.6 K 2024-11-20T12:15:59,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/407292de193944058be7ed62805e9698 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698 2024-11-20T12:15:59,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698, entries=150, sequenceid=69, filesize=11.7 K 2024-11-20T12:15:59,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/3bd133f7ed1f4131bd54966ad48c8353 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353 2024-11-20T12:15:59,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353, entries=150, sequenceid=69, filesize=11.7 K 2024-11-20T12:15:59,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for bdb95162a28369d7de140a1abe1b7035 in 1258ms, sequenceid=69, compaction requested=false 2024-11-20T12:15:59,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:59,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T12:15:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:59,514 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:15:59,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112061567032425d4125b1634704ec017fd2_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104958170/Put/seqid=0 2024-11-20T12:15:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T12:15:59,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742193_1369 (size=12154) 2024-11-20T12:15:59,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,531 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112061567032425d4125b1634704ec017fd2_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061567032425d4125b1634704ec017fd2_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:15:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6e6780bb694a474e885f095221380574, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:15:59,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6e6780bb694a474e885f095221380574 is 175, key is test_row_0/A:col10/1732104958170/Put/seqid=0 2024-11-20T12:15:59,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742194_1370 (size=30955) 2024-11-20T12:15:59,538 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6e6780bb694a474e885f095221380574 2024-11-20T12:15:59,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/4759ab4ec54a40c6b896bf00d8a53f6b is 50, key is test_row_0/B:col10/1732104958170/Put/seqid=0 2024-11-20T12:15:59,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742195_1371 (size=12001) 2024-11-20T12:15:59,565 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/4759ab4ec54a40c6b896bf00d8a53f6b 2024-11-20T12:15:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/a0945e6b915d4404955091f02979471c is 50, key is test_row_0/C:col10/1732104958170/Put/seqid=0 2024-11-20T12:15:59,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742196_1372 (size=12001) 2024-11-20T12:15:59,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T12:15:59,978 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/a0945e6b915d4404955091f02979471c 2024-11-20T12:15:59,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6e6780bb694a474e885f095221380574 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574 2024-11-20T12:15:59,986 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574, entries=150, sequenceid=84, filesize=30.2 K 2024-11-20T12:15:59,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/4759ab4ec54a40c6b896bf00d8a53f6b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b 2024-11-20T12:15:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,990 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b, entries=150, sequenceid=84, filesize=11.7 K 2024-11-20T12:15:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/a0945e6b915d4404955091f02979471c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c 2024-11-20T12:15:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,996 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c, entries=150, sequenceid=84, filesize=11.7 K 2024-11-20T12:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,997 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for bdb95162a28369d7de140a1abe1b7035 in 484ms, sequenceid=84, compaction requested=true 2024-11-20T12:15:59,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:15:59,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:15:59,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T12:15:59,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T12:15:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:15:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T12:16:00,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 790 msec 2024-11-20T12:16:00,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 795 msec 2024-11-20T12:16:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T12:16:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,347 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,347 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-20T12:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:00,349 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,350 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:00,503 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T12:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200b2d08d0d39648b3a5acdac75f0356a7_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104960495/Put/seqid=0 2024-11-20T12:16:00,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742197_1373 (size=14594) 2024-11-20T12:16:00,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:00,581 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200b2d08d0d39648b3a5acdac75f0356a7_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b2d08d0d39648b3a5acdac75f0356a7_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:00,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/b337849ee6d847e1914f263c3a1e68c8, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:00,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/b337849ee6d847e1914f263c3a1e68c8 is 175, key is test_row_0/A:col10/1732104960495/Put/seqid=0 2024-11-20T12:16:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742198_1374 (size=39549) 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104960560, max timestamp=1732104960610Num uniques 859; 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104960560, max timestamp=1732104960610Num uniques 859; 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104960560, max timestamp=1732104960610Num uniques 859; 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:00,613 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105020646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105020647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105020648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:00,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105020662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105020663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105020766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105020766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105020777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105020777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105020786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105020987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105020987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:00,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:00,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:00,998 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/b337849ee6d847e1914f263c3a1e68c8 2024-11-20T12:16:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f0e8ab9ed7264069b501ab6d5bfa30c3 is 50, key is test_row_0/B:col10/1732104960495/Put/seqid=0 2024-11-20T12:16:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105020996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105020996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742199_1375 (size=12001) 2024-11-20T12:16:01,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105021006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105021296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105021297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105021317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105021317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105021326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,417 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f0e8ab9ed7264069b501ab6d5bfa30c3 2024-11-20T12:16:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/54b209e05d2e428dab85f023fea79215 is 50, key is test_row_0/C:col10/1732104960495/Put/seqid=0 2024-11-20T12:16:01,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742200_1376 (size=12001) 2024-11-20T12:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:01,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105021817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105021817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,830 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/54b209e05d2e428dab85f023fea79215 2024-11-20T12:16:01,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/b337849ee6d847e1914f263c3a1e68c8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8 2024-11-20T12:16:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105021826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105021826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:01,837 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8, entries=200, sequenceid=93, filesize=38.6 K 2024-11-20T12:16:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f0e8ab9ed7264069b501ab6d5bfa30c3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3 2024-11-20T12:16:01,841 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T12:16:01,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/54b209e05d2e428dab85f023fea79215 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215 2024-11-20T12:16:01,844 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T12:16:01,845 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for bdb95162a28369d7de140a1abe1b7035 in 1342ms, sequenceid=93, compaction requested=true 2024-11-20T12:16:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:01,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-20T12:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-20T12:16:01,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T12:16:01,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-20T12:16:01,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.5020 sec 2024-11-20T12:16:01,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:01,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-20T12:16:01,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:01,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:01,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:01,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:01,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:01,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:01,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000739d287b1e403c8203583dffed7fa1_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:01,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742201_1377 (size=14644) 2024-11-20T12:16:01,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105021894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:01,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105022007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105022227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:02,267 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112000739d287b1e403c8203583dffed7fa1_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000739d287b1e403c8203583dffed7fa1_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:02,267 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/1a47303039fa467990d20bcac594a2f4, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:02,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/1a47303039fa467990d20bcac594a2f4 is 175, key is test_row_0/A:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:02,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742202_1378 (size=39599) 2024-11-20T12:16:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T12:16:02,497 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:02,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-20T12:16:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:02,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:02,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:02,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105022546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:02,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:02,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/1a47303039fa467990d20bcac594a2f4 2024-11-20T12:16:02,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c8cbe6e8b3f4d238ceb4734a6105053 is 50, key is test_row_0/B:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:02,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742203_1379 (size=12051) 2024-11-20T12:16:02,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105022837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105022847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105022847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105022847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:02,858 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,858 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,859 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:02,859 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:02,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:02,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:02,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:02,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105023066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:03,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:03,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c8cbe6e8b3f4d238ceb4734a6105053 2024-11-20T12:16:03,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1785b91d58d140849277462810708151 is 50, key is test_row_0/C:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:03,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742204_1380 (size=12051) 2024-11-20T12:16:03,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:03,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:03,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:03,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1785b91d58d140849277462810708151 2024-11-20T12:16:03,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/1a47303039fa467990d20bcac594a2f4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4 2024-11-20T12:16:03,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4, entries=200, sequenceid=129, filesize=38.7 K 2024-11-20T12:16:03,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c8cbe6e8b3f4d238ceb4734a6105053 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053 2024-11-20T12:16:03,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053, entries=150, sequenceid=129, filesize=11.8 K 2024-11-20T12:16:03,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1785b91d58d140849277462810708151 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151 2024-11-20T12:16:03,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151, entries=150, sequenceid=129, filesize=11.8 K 2024-11-20T12:16:03,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for bdb95162a28369d7de140a1abe1b7035 in 1667ms, sequenceid=129, compaction requested=true 2024-11-20T12:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:03,519 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:03,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:03,519 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:03,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:03,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 180710 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:03,521 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,521 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,521 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/116280921b3045aea77f81bfe3438aea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=58.7 K 2024-11-20T12:16:03,521 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=176.5 K 2024-11-20T12:16:03,521 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4] 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 116280921b3045aea77f81bfe3438aea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 407292de193944058be7ed62805e9698, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732104957459 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 585775ee23c647c7a093efacca43e2d4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:16:03,521 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4759ab4ec54a40c6b896bf00d8a53f6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732104958144 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e76139abcc34e3aa3b2184233f97c18, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732104957384 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e8ab9ed7264069b501ab6d5bfa30c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104960493 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e6780bb694a474e885f095221380574, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732104958144 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b337849ee6d847e1914f263c3a1e68c8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104960427 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c8cbe6e8b3f4d238ceb4734a6105053, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960661 2024-11-20T12:16:03,522 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a47303039fa467990d20bcac594a2f4, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960627 2024-11-20T12:16:03,536 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:03,539 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202ab10bbe87c74cc0ab6e97ee22314a0e_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:03,539 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#323 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:03,539 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/a8279a57ddae45c1bb3524db2ad2c0db is 50, key is test_row_0/B:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:03,542 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202ab10bbe87c74cc0ab6e97ee22314a0e_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:03,542 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202ab10bbe87c74cc0ab6e97ee22314a0e_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742205_1381 (size=12325) 2024-11-20T12:16:03,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742206_1382 (size=4469) 2024-11-20T12:16:03,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-20T12:16:03,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,580 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aee77018c66a4a51bcb0e4cae8ea0b61_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742207_1383 (size=12304) 2024-11-20T12:16:03,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:03,949 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/a8279a57ddae45c1bb3524db2ad2c0db as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/a8279a57ddae45c1bb3524db2ad2c0db 2024-11-20T12:16:03,952 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:03,953 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/c450941cdcbb415c9a5248c3ed6bef9d is 175, key is test_row_0/A:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:03,955 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into a8279a57ddae45c1bb3524db2ad2c0db(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:03,955 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:03,955 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=11, startTime=1732104963519; duration=0sec 2024-11-20T12:16:03,956 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:03,956 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:03,956 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:03,960 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:03,960 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:16:03,960 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:03,960 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/dd1353769cef4598b18e23aec2325cb4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=58.7 K 2024-11-20T12:16:03,961 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting dd1353769cef4598b18e23aec2325cb4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732104956286 2024-11-20T12:16:03,961 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bd133f7ed1f4131bd54966ad48c8353, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732104957459 2024-11-20T12:16:03,962 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a0945e6b915d4404955091f02979471c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732104958144 2024-11-20T12:16:03,962 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b209e05d2e428dab85f023fea79215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104960493 2024-11-20T12:16:03,963 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 1785b91d58d140849277462810708151, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960661 2024-11-20T12:16:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742208_1384 (size=31279) 2024-11-20T12:16:03,973 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:03,973 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/0b27fb5215154426829d98cc422ad1e4 is 50, key is test_row_0/C:col10/1732104961851/Put/seqid=0 2024-11-20T12:16:03,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742209_1385 (size=12325) 2024-11-20T12:16:03,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:03,994 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aee77018c66a4a51bcb0e4cae8ea0b61_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee77018c66a4a51bcb0e4cae8ea0b61_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:03,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0d9f7a9993f94b99ac487c5226ccd779, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:03,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0d9f7a9993f94b99ac487c5226ccd779 is 175, key is test_row_0/A:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:04,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742210_1386 (size=31105) 2024-11-20T12:16:04,004 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0d9f7a9993f94b99ac487c5226ccd779 2024-11-20T12:16:04,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/483090cd36eb4b4e9af2fce8aead6b2b is 50, key is test_row_0/B:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:04,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742211_1387 (size=12151) 2024-11-20T12:16:04,029 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/483090cd36eb4b4e9af2fce8aead6b2b 2024-11-20T12:16:04,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/41776f3494364adabd4ff10a975e7e15 is 50, key is test_row_0/C:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:04,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742212_1388 (size=12151) 2024-11-20T12:16:04,050 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/41776f3494364adabd4ff10a975e7e15 2024-11-20T12:16:04,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/0d9f7a9993f94b99ac487c5226ccd779 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779 2024-11-20T12:16:04,061 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779, entries=150, sequenceid=137, filesize=30.4 K 2024-11-20T12:16:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/483090cd36eb4b4e9af2fce8aead6b2b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b 2024-11-20T12:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,067 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T12:16:04,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/41776f3494364adabd4ff10a975e7e15 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,072 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15, entries=150, sequenceid=137, filesize=11.9 K 2024-11-20T12:16:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bdb95162a28369d7de140a1abe1b7035 in 494ms, sequenceid=137, compaction requested=false 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-20T12:16:04,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T12:16:04,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5740 sec 2024-11-20T12:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5820 sec 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,371 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/c450941cdcbb415c9a5248c3ed6bef9d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d 2024-11-20T12:16:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,379 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into c450941cdcbb415c9a5248c3ed6bef9d(size=30.5 K), total size for store is 60.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:04,379 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:04,380 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=11, startTime=1732104963519; duration=0sec 2024-11-20T12:16:04,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,380 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:04,380 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:04,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,384 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/0b27fb5215154426829d98cc422ad1e4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0b27fb5215154426829d98cc422ad1e4 2024-11-20T12:16:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,391 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into 0b27fb5215154426829d98cc422ad1e4(size=12.0 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:04,391 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:04,391 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=11, startTime=1732104963519; duration=0sec 2024-11-20T12:16:04,391 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:04,391 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:04,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:04,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-20T12:16:04,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-20T12:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,650 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f9be3a5b54474707b7808766054ca882_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_1/A:col10/1732104964580/Put/seqid=0 2024-11-20T12:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742213_1389 (size=12304) 2024-11-20T12:16:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,709 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f9be3a5b54474707b7808766054ca882_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f9be3a5b54474707b7808766054ca882_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,710 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/ce629732919047b3a9fd86cef4caa9af, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/ce629732919047b3a9fd86cef4caa9af is 175, key is test_row_1/A:col10/1732104964580/Put/seqid=0 2024-11-20T12:16:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:04,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742214_1390 (size=31101) 2024-11-20T12:16:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:04,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:04,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9558, totalHeapSize=21976, min timestamp=1732104964857, max timestamp=1732104964891Num uniques 859; 2024-11-20T12:16:04,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1732104964857, max timestamp=1732104964891Num uniques 859; 2024-11-20T12:16:04,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39328, new segment data size=9558, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:04,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8240, suffix heap size=117064,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=180 2024-11-20T12:16:04,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732104964857, max timestamp=1732104964891Num uniques 859; 2024-11-20T12:16:04,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:04,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105024927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:04,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105024933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:04,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105024934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105024936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:04,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105024937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:04,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:04,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:04,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:04,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:05,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105025047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105025047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105025057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105025057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105025057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,151 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/ce629732919047b3a9fd86cef4caa9af 2024-11-20T12:16:05,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5f05657522224b65a419da4fa626276c is 50, key is test_row_1/B:col10/1732104964580/Put/seqid=0 2024-11-20T12:16:05,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742215_1391 (size=9757) 2024-11-20T12:16:05,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5f05657522224b65a419da4fa626276c 2024-11-20T12:16:05,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/4d600cd0d1eb435c8e5c3a04e8c0e693 is 50, key is test_row_1/C:col10/1732104964580/Put/seqid=0 2024-11-20T12:16:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742216_1392 (size=9757) 2024-11-20T12:16:05,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105025267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:05,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105025269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105025277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105025277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105025283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/4d600cd0d1eb435c8e5c3a04e8c0e693 2024-11-20T12:16:05,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105025577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/ce629732919047b3a9fd86cef4caa9af as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af 2024-11-20T12:16:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105025586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af, entries=150, sequenceid=150, filesize=30.4 K 2024-11-20T12:16:05,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5f05657522224b65a419da4fa626276c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c 2024-11-20T12:16:05,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c, entries=100, sequenceid=150, filesize=9.5 K 2024-11-20T12:16:05,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/4d600cd0d1eb435c8e5c3a04e8c0e693 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693 2024-11-20T12:16:05,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693, entries=100, sequenceid=150, filesize=9.5 K 2024-11-20T12:16:05,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.89 KB/136076 for bdb95162a28369d7de140a1abe1b7035 in 960ms, sequenceid=150, compaction requested=true 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:05,603 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:05,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:05,603 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:05,603 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93485 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:05,603 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:05,604 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:05,604 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:05,604 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,604 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,604 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/a8279a57ddae45c1bb3524db2ad2c0db, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=33.4 K 2024-11-20T12:16:05,604 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=91.3 K 2024-11-20T12:16:05,604 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,604 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af] 2024-11-20T12:16:05,604 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c450941cdcbb415c9a5248c3ed6bef9d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960661 2024-11-20T12:16:05,604 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a8279a57ddae45c1bb3524db2ad2c0db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960661 2024-11-20T12:16:05,605 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 483090cd36eb4b4e9af2fce8aead6b2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732104961865 2024-11-20T12:16:05,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9f7a9993f94b99ac487c5226ccd779, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732104961865 2024-11-20T12:16:05,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce629732919047b3a9fd86cef4caa9af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104964501 2024-11-20T12:16:05,605 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f05657522224b65a419da4fa626276c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104964580 2024-11-20T12:16:05,610 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:05,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=139.60 KB heapSize=397.27 KB 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:05,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:05,618 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:05,619 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/b338d4d6063445279c2ea0d0237d509e is 50, key is test_row_0/B:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:05,623 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120837b58f138794a5b80981cc5cd9e809e_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:05,625 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120837b58f138794a5b80981cc5cd9e809e_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:05,626 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120837b58f138794a5b80981cc5cd9e809e_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:05,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d64a2c23a954b78a863543927fd2166_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104965609/Put/seqid=0 2024-11-20T12:16:05,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742219_1395 (size=14794) 2024-11-20T12:16:05,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742217_1393 (size=12527) 2024-11-20T12:16:05,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742218_1394 (size=4469) 2024-11-20T12:16:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105025631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105025631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105025633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105025747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105025747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105025756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:05,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:05,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:05,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:05,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105025967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105025967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105025967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:05,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:06,039 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#332 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:06,039 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cc8d115410584acca37abde2b9ff9fe1 is 175, key is test_row_0/A:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:06,046 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205d64a2c23a954b78a863543927fd2166_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d64a2c23a954b78a863543927fd2166_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:06,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742220_1396 (size=31588) 2024-11-20T12:16:06,047 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/b338d4d6063445279c2ea0d0237d509e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b338d4d6063445279c2ea0d0237d509e 2024-11-20T12:16:06,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f4bf372d6a0a45758568c53de8f9a4bd, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:06,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f4bf372d6a0a45758568c53de8f9a4bd is 175, key is test_row_0/A:col10/1732104965609/Put/seqid=0 2024-11-20T12:16:06,053 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/cc8d115410584acca37abde2b9ff9fe1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1 2024-11-20T12:16:06,055 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into b338d4d6063445279c2ea0d0237d509e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:06,055 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:06,055 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=13, startTime=1732104965603; duration=0sec 2024-11-20T12:16:06,055 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:06,055 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:06,055 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:06,056 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:06,056 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:16:06,056 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,056 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0b27fb5215154426829d98cc422ad1e4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=33.4 K 2024-11-20T12:16:06,056 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b27fb5215154426829d98cc422ad1e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732104960661 2024-11-20T12:16:06,057 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 41776f3494364adabd4ff10a975e7e15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732104961865 2024-11-20T12:16:06,057 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d600cd0d1eb435c8e5c3a04e8c0e693, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104964580 2024-11-20T12:16:06,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742221_1397 (size=39749) 2024-11-20T12:16:06,058 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into cc8d115410584acca37abde2b9ff9fe1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:06,058 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:06,058 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=13, startTime=1732104965603; duration=0sec 2024-11-20T12:16:06,058 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:06,058 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:06,065 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f4bf372d6a0a45758568c53de8f9a4bd 2024-11-20T12:16:06,067 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:06,068 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/afd687a2ebc7486aa8fb09724e131a01 is 50, key is test_row_0/C:col10/1732104961865/Put/seqid=0 2024-11-20T12:16:06,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742222_1398 (size=12527) 2024-11-20T12:16:06,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ab092242c9bf435689faa1cd961aa68b is 50, key is test_row_0/B:col10/1732104965609/Put/seqid=0 2024-11-20T12:16:06,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742223_1399 (size=12151) 2024-11-20T12:16:06,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105026096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105026106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105026277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105026287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105026287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,475 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/afd687a2ebc7486aa8fb09724e131a01 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/afd687a2ebc7486aa8fb09724e131a01 2024-11-20T12:16:06,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ab092242c9bf435689faa1cd961aa68b 2024-11-20T12:16:06,478 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into afd687a2ebc7486aa8fb09724e131a01(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:06,478 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:06,478 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=13, startTime=1732104965603; duration=0sec 2024-11-20T12:16:06,479 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:06,479 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:06,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/7f2c9a1d76b74d55a92e7f6e319f41ac is 50, key is test_row_0/C:col10/1732104965609/Put/seqid=0 2024-11-20T12:16:06,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742224_1400 (size=12151) 2024-11-20T12:16:06,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105026787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:06,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105026807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105026807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:06,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:06,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.67 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/7f2c9a1d76b74d55a92e7f6e319f41ac 2024-11-20T12:16:06,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f4bf372d6a0a45758568c53de8f9a4bd as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd 2024-11-20T12:16:06,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd, entries=200, sequenceid=187, filesize=38.8 K 2024-11-20T12:16:06,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ab092242c9bf435689faa1cd961aa68b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b 2024-11-20T12:16:06,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b, entries=150, sequenceid=187, filesize=11.9 K 2024-11-20T12:16:06,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/7f2c9a1d76b74d55a92e7f6e319f41ac as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac 2024-11-20T12:16:06,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac, entries=150, sequenceid=187, filesize=11.9 K 2024-11-20T12:16:06,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~139.60 KB/142946, heapSize ~397.23 KB/406760, currentSize=40.25 KB/41220 for bdb95162a28369d7de140a1abe1b7035 in 1308ms, sequenceid=187, compaction requested=false 2024-11-20T12:16:06,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:06,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:06,950 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:06,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112076009d71bb9749059511fd7d929865fb_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104965617/Put/seqid=0 2024-11-20T12:16:06,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742225_1401 (size=12304) 2024-11-20T12:16:07,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:07,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:07,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104967182, max timestamp=1732104967216Num uniques 859; 2024-11-20T12:16:07,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104967182, max timestamp=1732104967216Num uniques 859; 2024-11-20T12:16:07,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:07,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:07,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104967182, max timestamp=1732104967216Num uniques 859; 2024-11-20T12:16:07,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105027318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105027326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:07,372 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112076009d71bb9749059511fd7d929865fb_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076009d71bb9749059511fd7d929865fb_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:07,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/e25067c63ff3464d9469c0ca693ce893, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:07,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/e25067c63ff3464d9469c0ca693ce893 is 175, key is test_row_0/A:col10/1732104965617/Put/seqid=0 2024-11-20T12:16:07,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742226_1402 (size=31105) 2024-11-20T12:16:07,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105027436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105027457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105027656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105027677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,779 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/e25067c63ff3464d9469c0ca693ce893 2024-11-20T12:16:07,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/6a268f43874d42cc9b6b416b6ba4ee97 is 50, key is test_row_0/B:col10/1732104965617/Put/seqid=0 2024-11-20T12:16:07,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742227_1403 (size=12151) 2024-11-20T12:16:07,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105027796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,811 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105027816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,828 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105027827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,839 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:07,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105027967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:07,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:08,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105027996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:08,204 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/6a268f43874d42cc9b6b416b6ba4ee97 2024-11-20T12:16:08,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8676d66b46a2412cba801c5c4f1b071c is 50, key is test_row_0/C:col10/1732104965617/Put/seqid=0 2024-11-20T12:16:08,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742228_1404 (size=12151) 2024-11-20T12:16:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105028487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105028516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:08,622 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8676d66b46a2412cba801c5c4f1b071c 2024-11-20T12:16:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/e25067c63ff3464d9469c0ca693ce893 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893 2024-11-20T12:16:08,628 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893, entries=150, sequenceid=199, filesize=30.4 K 2024-11-20T12:16:08,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/6a268f43874d42cc9b6b416b6ba4ee97 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97 2024-11-20T12:16:08,632 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T12:16:08,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8676d66b46a2412cba801c5c4f1b071c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c 2024-11-20T12:16:08,635 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c, entries=150, sequenceid=199, filesize=11.9 K 2024-11-20T12:16:08,636 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for bdb95162a28369d7de140a1abe1b7035 in 1686ms, sequenceid=199, compaction requested=true 2024-11-20T12:16:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-20T12:16:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-20T12:16:08,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-20T12:16:08,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.9870 sec 2024-11-20T12:16:08,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.9930 sec 2024-11-20T12:16:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T12:16:08,827 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:08,827 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-20T12:16:08,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T12:16:08,829 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:08,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:08,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T12:16:08,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-20T12:16:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:08,984 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112080ca1e7ab2c840219d19fade0ffd5d62_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104967322/Put/seqid=0 2024-11-20T12:16:09,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742229_1405 (size=12304) 2024-11-20T12:16:09,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,011 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112080ca1e7ab2c840219d19fade0ffd5d62_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112080ca1e7ab2c840219d19fade0ffd5d62_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:09,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/be9d2c05d37e45e5ac63cfab3e0dd60b, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:09,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/be9d2c05d37e45e5ac63cfab3e0dd60b is 175, key is test_row_0/A:col10/1732104967322/Put/seqid=0 2024-11-20T12:16:09,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742230_1406 (size=31105) 2024-11-20T12:16:09,022 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/be9d2c05d37e45e5ac63cfab3e0dd60b 2024-11-20T12:16:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/96c8fd3f8ebc48e69b884aa95ca48799 is 50, key is test_row_0/B:col10/1732104967322/Put/seqid=0 2024-11-20T12:16:09,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742231_1407 (size=12151) 2024-11-20T12:16:09,040 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/96c8fd3f8ebc48e69b884aa95ca48799 2024-11-20T12:16:09,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/5a32c7ba86344fcc8bd1806e1a548ca0 is 50, key is test_row_0/C:col10/1732104967322/Put/seqid=0 2024-11-20T12:16:09,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742232_1408 (size=12151) 2024-11-20T12:16:09,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T12:16:09,449 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/5a32c7ba86344fcc8bd1806e1a548ca0 2024-11-20T12:16:09,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/be9d2c05d37e45e5ac63cfab3e0dd60b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b 2024-11-20T12:16:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T12:16:09,458 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b, entries=150, sequenceid=236, filesize=30.4 K 2024-11-20T12:16:09,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/96c8fd3f8ebc48e69b884aa95ca48799 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799 2024-11-20T12:16:09,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,464 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:16:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/5a32c7ba86344fcc8bd1806e1a548ca0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,469 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:16:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=0 B/0 for bdb95162a28369d7de140a1abe1b7035 in 486ms, sequenceid=236, compaction requested=true 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:09,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:09,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T12:16:09,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 642 msec 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 647 msec 2024-11-20T12:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:09,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:09,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:09,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bad36962b15c4d8a8c7a3f792e938316_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:09,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742233_1409 (size=14794) 2024-11-20T12:16:09,723 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:09,728 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bad36962b15c4d8a8c7a3f792e938316_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bad36962b15c4d8a8c7a3f792e938316_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:09,729 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/725e53b1c37448d7a1182438c2cd76b0, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:09,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/725e53b1c37448d7a1182438c2cd76b0 is 175, key is test_row_0/A:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:09,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742234_1410 (size=39749) 2024-11-20T12:16:09,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104969740, max timestamp=1732104969782Num uniques 859; 2024-11-20T12:16:09,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:09,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104969740, max timestamp=1732104969782Num uniques 859; 2024-11-20T12:16:09,784 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:09,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104969740, max timestamp=1732104969782Num uniques 859; 2024-11-20T12:16:09,785 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:09,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105029847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105029848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105029856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,864 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105029862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105029862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-20T12:16:09,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:09,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T12:16:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:09,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:09,970 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:09,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:09,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105029967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105029967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105029977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:09,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105029977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:09,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:10,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T12:16:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/725e53b1c37448d7a1182438c2cd76b0 2024-11-20T12:16:10,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f6c0a732acb64fb382edf3dc8c4649a6 is 50, key is test_row_0/B:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:10,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742235_1411 (size=12151) 2024-11-20T12:16:10,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105030177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105030177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105030196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105030196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T12:16:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:10,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T12:16:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:10,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105030497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105030497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105030506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105030506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:10,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:10,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f6c0a732acb64fb382edf3dc8c4649a6 2024-11-20T12:16:10,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/080e19de0c324e3ead28c838ed62d9b0 is 50, key is test_row_0/C:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:10,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742236_1412 (size=12151) 2024-11-20T12:16:10,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/080e19de0c324e3ead28c838ed62d9b0 2024-11-20T12:16:10,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T12:16:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:10,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:10,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/725e53b1c37448d7a1182438c2cd76b0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0 2024-11-20T12:16:10,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0, entries=200, sequenceid=246, filesize=38.8 K 2024-11-20T12:16:10,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f6c0a732acb64fb382edf3dc8c4649a6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6 2024-11-20T12:16:10,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6, entries=150, sequenceid=246, filesize=11.9 K 2024-11-20T12:16:10,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/080e19de0c324e3ead28c838ed62d9b0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0 2024-11-20T12:16:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:10,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0, entries=150, sequenceid=246, filesize=11.9 K 2024-11-20T12:16:10,597 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bdb95162a28369d7de140a1abe1b7035 in 917ms, sequenceid=246, compaction requested=true 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:10,597 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:10,597 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:10,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:10,598 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:10,598 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173296 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:10,598 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:10,598 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:10,598 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,598 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,599 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b338d4d6063445279c2ea0d0237d509e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=59.7 K 2024-11-20T12:16:10,599 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=169.2 K 2024-11-20T12:16:10,599 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,599 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0] 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc8d115410584acca37abde2b9ff9fe1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104961865 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b338d4d6063445279c2ea0d0237d509e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104961865 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4bf372d6a0a45758568c53de8f9a4bd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732104964932 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ab092242c9bf435689faa1cd961aa68b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732104964932 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25067c63ff3464d9469c0ca693ce893, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732104965617 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a268f43874d42cc9b6b416b6ba4ee97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732104965617 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c8fd3f8ebc48e69b884aa95ca48799, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104967300 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting be9d2c05d37e45e5ac63cfab3e0dd60b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104967300 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c0a732acb64fb382edf3dc8c4649a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969640 2024-11-20T12:16:10,600 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725e53b1c37448d7a1182438c2cd76b0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969558 2024-11-20T12:16:10,606 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:10,609 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#347 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:10,610 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/9514b0c80fa64754aad022806a7fa107 is 50, key is test_row_0/B:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:10,611 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120bad889b1a23746728c6518fe5ae956e8_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:10,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742237_1413 (size=12697) 2024-11-20T12:16:10,614 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120bad889b1a23746728c6518fe5ae956e8_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:10,614 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bad889b1a23746728c6518fe5ae956e8_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:10,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742238_1414 (size=4469) 2024-11-20T12:16:10,623 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#348 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:10,624 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/3aeea1c7a7aa43ababa65eb085cc6544 is 175, key is test_row_0/A:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:10,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742239_1415 (size=31651) 2024-11-20T12:16:10,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:10,733 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:10,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:10,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120656a9bfc4b514a20a2a2a4e33d615456_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104969845/Put/seqid=0 2024-11-20T12:16:10,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742240_1416 (size=12454) 2024-11-20T12:16:11,018 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/9514b0c80fa64754aad022806a7fa107 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9514b0c80fa64754aad022806a7fa107 2024-11-20T12:16:11,022 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into 9514b0c80fa64754aad022806a7fa107(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:11,022 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:11,022 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=11, startTime=1732104970597; duration=0sec 2024-11-20T12:16:11,022 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:11,022 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:11,022 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T12:16:11,023 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T12:16:11,023 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:16:11,024 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:11,024 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/afd687a2ebc7486aa8fb09724e131a01, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=59.7 K 2024-11-20T12:16:11,024 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting afd687a2ebc7486aa8fb09724e131a01, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732104961865 2024-11-20T12:16:11,024 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2c9a1d76b74d55a92e7f6e319f41ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732104964932 2024-11-20T12:16:11,024 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8676d66b46a2412cba801c5c4f1b071c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732104965617 2024-11-20T12:16:11,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a32c7ba86344fcc8bd1806e1a548ca0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104967300 2024-11-20T12:16:11,025 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 080e19de0c324e3ead28c838ed62d9b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969640 2024-11-20T12:16:11,034 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/3aeea1c7a7aa43ababa65eb085cc6544 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544 2024-11-20T12:16:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:11,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:11,039 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into 3aeea1c7a7aa43ababa65eb085cc6544(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:11,039 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:11,039 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=11, startTime=1732104970597; duration=0sec 2024-11-20T12:16:11,039 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:11,039 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:11,041 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#350 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:11,041 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/a36d1650a5e94c51a666216fb4b4aef3 is 50, key is test_row_0/C:col10/1732104969678/Put/seqid=0 2024-11-20T12:16:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742241_1417 (size=12697) 2024-11-20T12:16:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105031054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105031056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105031057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105031058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:11,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:11,146 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120656a9bfc4b514a20a2a2a4e33d615456_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120656a9bfc4b514a20a2a2a4e33d615456_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/00de47c0d99d4459b786a558feb59f23, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:11,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/00de47c0d99d4459b786a558feb59f23 is 175, key is test_row_0/A:col10/1732104969845/Put/seqid=0 2024-11-20T12:16:11,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742242_1418 (size=31255) 2024-11-20T12:16:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105031166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105031166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105031177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105031177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105031377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105031387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105031396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105031397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,453 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/a36d1650a5e94c51a666216fb4b4aef3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a36d1650a5e94c51a666216fb4b4aef3 2024-11-20T12:16:11,456 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into a36d1650a5e94c51a666216fb4b4aef3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:11,456 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:11,456 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=11, startTime=1732104970597; duration=0sec 2024-11-20T12:16:11,456 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:11,456 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:11,550 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/00de47c0d99d4459b786a558feb59f23 2024-11-20T12:16:11,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e773f12a31ff4d189b5a84a95f137f30 is 50, key is test_row_0/B:col10/1732104969845/Put/seqid=0 2024-11-20T12:16:11,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742243_1419 (size=12301) 2024-11-20T12:16:11,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105031687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105031707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105031707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105031707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:11,960 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e773f12a31ff4d189b5a84a95f137f30 2024-11-20T12:16:11,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e7edb1b5b66b4aeba55b4fd625f26dc0 is 50, key is test_row_0/C:col10/1732104969845/Put/seqid=0 2024-11-20T12:16:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742244_1420 (size=12301) 2024-11-20T12:16:12,003 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e7edb1b5b66b4aeba55b4fd625f26dc0 2024-11-20T12:16:12,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/00de47c0d99d4459b786a558feb59f23 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23 2024-11-20T12:16:12,013 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23, entries=150, sequenceid=282, filesize=30.5 K 2024-11-20T12:16:12,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e773f12a31ff4d189b5a84a95f137f30 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30 2024-11-20T12:16:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,021 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30, entries=150, sequenceid=282, filesize=12.0 K 2024-11-20T12:16:12,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e7edb1b5b66b4aeba55b4fd625f26dc0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0 2024-11-20T12:16:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,031 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0, entries=150, sequenceid=282, filesize=12.0 K 2024-11-20T12:16:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,031 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for bdb95162a28369d7de140a1abe1b7035 in 1298ms, sequenceid=282, compaction requested=false 2024-11-20T12:16:12,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T12:16:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T12:16:12,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T12:16:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0630 sec 2024-11-20T12:16:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.0700 sec 2024-11-20T12:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T12:16:12,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:12,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T12:16:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,121 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:12,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:12,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c1489df05894eb3b27d68255e80c77a_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742245_1421 (size=12454) 2024-11-20T12:16:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105032344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105032345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105032346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105032396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105032507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105032507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105032507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105032516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,701 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:12,706 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206c1489df05894eb3b27d68255e80c77a_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c1489df05894eb3b27d68255e80c77a_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:12,707 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/d4990c05b9c24090b19f72ed02738135, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:12,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/d4990c05b9c24090b19f72ed02738135 is 175, key is test_row_0/A:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:12,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742246_1422 (size=31251) 2024-11-20T12:16:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105032737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105032737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105032747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105032748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:12,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:12,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:12,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105033057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105033057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105033066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105033067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,148 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/d4990c05b9c24090b19f72ed02738135 2024-11-20T12:16:13,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ce1d713a5a2b4e5db621f49641a2fbdd is 50, key is test_row_0/B:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:13,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742247_1423 (size=9857) 2024-11-20T12:16:13,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ce1d713a5a2b4e5db621f49641a2fbdd 2024-11-20T12:16:13,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/0a80c185e36e4f0c82f47255150e268e is 50, key is test_row_0/C:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:13,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:13,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742248_1424 (size=9857) 2024-11-20T12:16:13,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/0a80c185e36e4f0c82f47255150e268e 2024-11-20T12:16:13,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/d4990c05b9c24090b19f72ed02738135 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135 2024-11-20T12:16:13,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135, entries=150, sequenceid=297, filesize=30.5 K 2024-11-20T12:16:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/ce1d713a5a2b4e5db621f49641a2fbdd as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd 2024-11-20T12:16:13,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd, entries=100, sequenceid=297, filesize=9.6 K 2024-11-20T12:16:13,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/0a80c185e36e4f0c82f47255150e268e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e 2024-11-20T12:16:13,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e, entries=100, sequenceid=297, filesize=9.6 K 2024-11-20T12:16:13,248 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bdb95162a28369d7de140a1abe1b7035 in 1011ms, sequenceid=297, compaction requested=true 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:13,248 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:13,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T12:16:13,249 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:13,272 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94157 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:13,272 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:13,272 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:13,272 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:13,273 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,273 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,273 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=92.0 K 2024-11-20T12:16:13,273 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9514b0c80fa64754aad022806a7fa107, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=34.0 K 2024-11-20T12:16:13,273 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,273 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135] 2024-11-20T12:16:13,275 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9514b0c80fa64754aad022806a7fa107, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969640 2024-11-20T12:16:13,275 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e773f12a31ff4d189b5a84a95f137f30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732104969786 2024-11-20T12:16:13,275 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aeea1c7a7aa43ababa65eb085cc6544, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969640 2024-11-20T12:16:13,275 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1d713a5a2b4e5db621f49641a2fbdd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732104971055 2024-11-20T12:16:13,276 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00de47c0d99d4459b786a558feb59f23, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732104969786 2024-11-20T12:16:13,278 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4990c05b9c24090b19f72ed02738135, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732104971054 2024-11-20T12:16:13,285 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:13,285 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/9f5e8c6ae4244de78d51d1729928a12f is 50, key is test_row_0/B:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:13,288 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,294 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120dc12ac066c7d40caa9a34b7e214e66d4_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,296 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120dc12ac066c7d40caa9a34b7e214e66d4_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,296 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dc12ac066c7d40caa9a34b7e214e66d4_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742250_1426 (size=4469) 2024-11-20T12:16:13,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742249_1425 (size=12949) 2024-11-20T12:16:13,349 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#357 average throughput is 0.40 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:13,349 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f734881a8fb24761aa30b471b928a3bf is 175, key is test_row_0/A:col10/1732104972236/Put/seqid=0 2024-11-20T12:16:13,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T12:16:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,354 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:13,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742251_1427 (size=32010) 2024-11-20T12:16:13,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba3b612fbcf34d21bc7560d090b2e880_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104972336/Put/seqid=0 2024-11-20T12:16:13,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742252_1428 (size=12454) 2024-11-20T12:16:13,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:13,487 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ba3b612fbcf34d21bc7560d090b2e880_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3b612fbcf34d21bc7560d090b2e880_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/639eb955da7545ae9d0c2b33622245ac, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/639eb955da7545ae9d0c2b33622245ac is 175, key is test_row_0/A:col10/1732104972336/Put/seqid=0 2024-11-20T12:16:13,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742253_1429 (size=31255) 2024-11-20T12:16:13,518 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/639eb955da7545ae9d0c2b33622245ac 2024-11-20T12:16:13,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e44790f8fba1412eb3e5433113522442 is 50, key is test_row_0/B:col10/1732104972336/Put/seqid=0 2024-11-20T12:16:13,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742254_1430 (size=12301) 2024-11-20T12:16:13,559 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e44790f8fba1412eb3e5433113522442 2024-11-20T12:16:13,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1fafef4d2ab94d2ca9a787a83cd23dda is 50, key is test_row_0/C:col10/1732104972336/Put/seqid=0 2024-11-20T12:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:13,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:13,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742255_1431 (size=12301) 2024-11-20T12:16:13,654 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1fafef4d2ab94d2ca9a787a83cd23dda 2024-11-20T12:16:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/639eb955da7545ae9d0c2b33622245ac as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac 2024-11-20T12:16:13,667 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac, entries=150, sequenceid=317, filesize=30.5 K 2024-11-20T12:16:13,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/e44790f8fba1412eb3e5433113522442 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442 2024-11-20T12:16:13,673 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442, entries=150, sequenceid=317, filesize=12.0 K 2024-11-20T12:16:13,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/1fafef4d2ab94d2ca9a787a83cd23dda as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda 2024-11-20T12:16:13,680 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda, entries=150, sequenceid=317, filesize=12.0 K 2024-11-20T12:16:13,682 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=53.67 KB/54960 for bdb95162a28369d7de140a1abe1b7035 in 328ms, sequenceid=317, compaction requested=true 2024-11-20T12:16:13,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:13,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T12:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T12:16:13,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T12:16:13,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-20T12:16:13,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.5680 sec 2024-11-20T12:16:13,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:13,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:13,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201592b78f9c24471caab68538fcfb08c4_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:13,743 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/9f5e8c6ae4244de78d51d1729928a12f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9f5e8c6ae4244de78d51d1729928a12f 2024-11-20T12:16:13,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742256_1432 (size=12454) 2024-11-20T12:16:13,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:13,749 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into 9f5e8c6ae4244de78d51d1729928a12f(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:13,749 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:13,749 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=13, startTime=1732104973248; duration=0sec 2024-11-20T12:16:13,749 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:13,749 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:13,749 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:13,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201592b78f9c24471caab68538fcfb08c4_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201592b78f9c24471caab68538fcfb08c4_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:13,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/a85649f91f2c45c295bf3b2e77cecd67, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:13,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/a85649f91f2c45c295bf3b2e77cecd67 is 175, key is test_row_0/A:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:13,755 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47156 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:13,757 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:16:13,757 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:13,757 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a36d1650a5e94c51a666216fb4b4aef3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=46.1 K 2024-11-20T12:16:13,758 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a36d1650a5e94c51a666216fb4b4aef3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732104969640 2024-11-20T12:16:13,758 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e7edb1b5b66b4aeba55b4fd625f26dc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732104969786 2024-11-20T12:16:13,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742257_1433 (size=31255) 2024-11-20T12:16:13,759 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a80c185e36e4f0c82f47255150e268e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732104971055 2024-11-20T12:16:13,760 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fafef4d2ab94d2ca9a787a83cd23dda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732104972334 2024-11-20T12:16:13,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/a85649f91f2c45c295bf3b2e77cecd67 2024-11-20T12:16:13,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c0f575c254f43d3b288ad8404bfa2fe is 50, key is test_row_0/B:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:13,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105033761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,790 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/f734881a8fb24761aa30b471b928a3bf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf 2024-11-20T12:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105033762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,795 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:13,795 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into f734881a8fb24761aa30b471b928a3bf(size=31.3 K), total size for store is 61.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:13,795 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:13,795 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/c061d7c62a6d4e7b91e58cec02408be3 is 50, key is test_row_0/C:col10/1732104972336/Put/seqid=0 2024-11-20T12:16:13,795 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=13, startTime=1732104973248; duration=0sec 2024-11-20T12:16:13,796 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:13,796 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:13,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105033788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105033791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742258_1434 (size=12301) 2024-11-20T12:16:13,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742259_1435 (size=12983) 2024-11-20T12:16:13,851 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/c061d7c62a6d4e7b91e58cec02408be3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c061d7c62a6d4e7b91e58cec02408be3 2024-11-20T12:16:13,856 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into c061d7c62a6d4e7b91e58cec02408be3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:13,856 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:13,857 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=12, startTime=1732104973248; duration=0sec 2024-11-20T12:16:13,857 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:13,857 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:13,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47680 deadline: 1732105033867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,871 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bdb95162a28369d7de140a1abe1b7035 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105033897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105033897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105033917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105033917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105034106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105034117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105034127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105034130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c0f575c254f43d3b288ad8404bfa2fe 2024-11-20T12:16:14,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8c135a5b6e074a43a570f0b0cf02fb6e is 50, key is test_row_0/C:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:14,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T12:16:14,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:14,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T12:16:14,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:14,277 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:14,278 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:14,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:14,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742260_1436 (size=12301) 2024-11-20T12:16:14,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8c135a5b6e074a43a570f0b0cf02fb6e 2024-11-20T12:16:14,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/a85649f91f2c45c295bf3b2e77cecd67 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67 2024-11-20T12:16:14,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67, entries=150, sequenceid=331, filesize=30.5 K 2024-11-20T12:16:14,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/5c0f575c254f43d3b288ad8404bfa2fe as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe 2024-11-20T12:16:14,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T12:16:14,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/8c135a5b6e074a43a570f0b0cf02fb6e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e 2024-11-20T12:16:14,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T12:16:14,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for bdb95162a28369d7de140a1abe1b7035 in 617ms, sequenceid=331, compaction requested=true 2024-11-20T12:16:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:14,310 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:14,310 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:14,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:14,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:14,311 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94520 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:14,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:14,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:14,311 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:14,311 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:14,312 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=92.3 K 2024-11-20T12:16:14,312 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:14,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67] 2024-11-20T12:16:14,312 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:14,312 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:14,312 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:14,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f734881a8fb24761aa30b471b928a3bf, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732104969786 2024-11-20T12:16:14,312 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9f5e8c6ae4244de78d51d1729928a12f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=36.7 K 2024-11-20T12:16:14,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5e8c6ae4244de78d51d1729928a12f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1732104969786 2024-11-20T12:16:14,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 639eb955da7545ae9d0c2b33622245ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732104972334 2024-11-20T12:16:14,313 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e44790f8fba1412eb3e5433113522442, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732104972334 2024-11-20T12:16:14,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85649f91f2c45c295bf3b2e77cecd67, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732104973669 2024-11-20T12:16:14,314 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c0f575c254f43d3b288ad8404bfa2fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732104973669 2024-11-20T12:16:14,338 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:14,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/3d06c501cfd64c348fc0c17a1a23bac3 is 50, key is test_row_0/B:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:14,352 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:14,373 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120eb4d1bcdd89c4792a782edd49bceb1f1_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:14,375 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120eb4d1bcdd89c4792a782edd49bceb1f1_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:14,375 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eb4d1bcdd89c4792a782edd49bceb1f1_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:14,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:14,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742261_1437 (size=13051) 2024-11-20T12:16:14,426 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/3d06c501cfd64c348fc0c17a1a23bac3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/3d06c501cfd64c348fc0c17a1a23bac3 2024-11-20T12:16:14,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742262_1438 (size=4469) 2024-11-20T12:16:14,431 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#366 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:14,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T12:16:14,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:14,431 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/63848bbc8c714281b9cc6731cbacfb6b is 175, key is test_row_0/A:col10/1732104973693/Put/seqid=0 2024-11-20T12:16:14,432 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:14,436 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into 3d06c501cfd64c348fc0c17a1a23bac3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:14,436 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:14,436 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=13, startTime=1732104974310; duration=0sec 2024-11-20T12:16:14,437 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:14,437 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:14,438 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T12:16:14,439 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T12:16:14,439 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T12:16:14,439 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. because compaction request was cancelled 2024-11-20T12:16:14,439 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:14,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. as already flushing 2024-11-20T12:16:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:14,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742263_1439 (size=32005) 2024-11-20T12:16:14,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112094533680854e46acaf1337159e3db9b2_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104973785/Put/seqid=0 2024-11-20T12:16:14,473 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/63848bbc8c714281b9cc6731cbacfb6b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b 2024-11-20T12:16:14,480 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into 63848bbc8c714281b9cc6731cbacfb6b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:14,480 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:14,480 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=13, startTime=1732104974310; duration=0sec 2024-11-20T12:16:14,481 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:14,481 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742264_1440 (size=12454) 2024-11-20T12:16:14,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105034486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105034492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105034493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105034501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,516 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112094533680854e46acaf1337159e3db9b2_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112094533680854e46acaf1337159e3db9b2_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/9bd375a209d144a1a0c310d2d38a4da0, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:14,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/9bd375a209d144a1a0c310d2d38a4da0 is 175, key is test_row_0/A:col10/1732104973785/Put/seqid=0 2024-11-20T12:16:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742265_1441 (size=31255) 2024-11-20T12:16:14,559 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/9bd375a209d144a1a0c310d2d38a4da0 2024-11-20T12:16:14,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f72f667c09de484494b8875419051aea is 50, key is test_row_0/B:col10/1732104973785/Put/seqid=0 2024-11-20T12:16:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105034617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105034617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105034627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105034627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,635 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742266_1442 (size=12301) 2024-11-20T12:16:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105034837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105034837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105034837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105034847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:15,045 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f72f667c09de484494b8875419051aea 2024-11-20T12:16:15,048 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-20T12:16:15,048 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,048 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,048 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/c3e7f62cf142484393e605990aea6025 is 50, key is test_row_0/C:col10/1732104973785/Put/seqid=0 2024-11-20T12:16:15,054 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-20T12:16:15,054 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,054 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-20T12:16:15,054 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,054 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,054 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,054 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,054 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742267_1443 (size=12301) 2024-11-20T12:16:15,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,059 INFO [Thread-1592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1592. 2024-11-20T12:16:15,059 DEBUG [Thread-1592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,059 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,059 DEBUG [Thread-1592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,065 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-20T12:16:15,066 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,066 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,066 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47726 deadline: 1732105035147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47710 deadline: 1732105035146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47702 deadline: 1732105035147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:15,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:15,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:47746 deadline: 1732105035167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:15,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:15,453 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/c3e7f62cf142484393e605990aea6025 2024-11-20T12:16:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/9bd375a209d144a1a0c310d2d38a4da0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0 2024-11-20T12:16:15,460 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0, entries=150, sequenceid=352, filesize=30.5 K 2024-11-20T12:16:15,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/f72f667c09de484494b8875419051aea as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea 2024-11-20T12:16:15,464 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea, entries=150, sequenceid=352, filesize=12.0 K 2024-11-20T12:16:15,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/c3e7f62cf142484393e605990aea6025 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025 2024-11-20T12:16:15,468 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025, entries=150, sequenceid=352, filesize=12.0 K 2024-11-20T12:16:15,469 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for bdb95162a28369d7de140a1abe1b7035 in 1038ms, sequenceid=352, compaction requested=true 2024-11-20T12:16:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:15,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T12:16:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T12:16:15,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T12:16:15,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1920 sec 2024-11-20T12:16:15,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.2040 sec 2024-11-20T12:16:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:15,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T12:16:15,661 INFO [Thread-1579 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1579. 2024-11-20T12:16:15,661 INFO [Thread-1581 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1581. 2024-11-20T12:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:15,661 DEBUG [Thread-1579 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:15,661 DEBUG [Thread-1581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:15,661 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,661 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:15,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:15,662 DEBUG [Thread-1581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,662 DEBUG [Thread-1579 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,662 INFO [Thread-1577 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1577. 2024-11-20T12:16:15,662 DEBUG [Thread-1577 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,662 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,662 DEBUG [Thread-1577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:15,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d46b73a399794d1ab351501f6d94ec43_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:15,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742268_1444 (size=12454) 2024-11-20T12:16:15,678 INFO [Thread-1585 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1585. 2024-11-20T12:16:15,678 DEBUG [Thread-1585 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:15,678 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,678 DEBUG [Thread-1585 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:15,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:16,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:16,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d46b73a399794d1ab351501f6d94ec43_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d46b73a399794d1ab351501f6d94ec43_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:16,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/8926f555bbe1433597b4e10139d4aae7, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:16,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/8926f555bbe1433597b4e10139d4aae7 is 175, key is test_row_0/A:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:16,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742269_1445 (size=31255) 2024-11-20T12:16:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T12:16:16,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:16,483 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=371, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/8926f555bbe1433597b4e10139d4aae7 2024-11-20T12:16:16,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/b52127585e0046d8a94261e2472d80e6 is 50, key is test_row_0/B:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:16,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742270_1446 (size=12301) 2024-11-20T12:16:16,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/b52127585e0046d8a94261e2472d80e6 2024-11-20T12:16:16,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/525aba15670a41de90ccaf7b6a7f2793 is 50, key is test_row_0/C:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:16,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742271_1447 (size=12301) 2024-11-20T12:16:17,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=371 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/525aba15670a41de90ccaf7b6a7f2793 2024-11-20T12:16:17,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/8926f555bbe1433597b4e10139d4aae7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7 2024-11-20T12:16:17,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7, entries=150, sequenceid=371, filesize=30.5 K 2024-11-20T12:16:17,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/b52127585e0046d8a94261e2472d80e6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6 2024-11-20T12:16:17,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6, entries=150, sequenceid=371, filesize=12.0 K 2024-11-20T12:16:17,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/525aba15670a41de90ccaf7b6a7f2793 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793 2024-11-20T12:16:17,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793, entries=150, sequenceid=371, filesize=12.0 K 2024-11-20T12:16:17,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for bdb95162a28369d7de140a1abe1b7035 in 1677ms, sequenceid=371, compaction requested=true 2024-11-20T12:16:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:17,337 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:17,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:17,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bdb95162a28369d7de140a1abe1b7035:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:17,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:17,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:17,338 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:17,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:17,338 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/A is initiating minor compaction (all files) 2024-11-20T12:16:17,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/B is initiating minor compaction (all files) 2024-11-20T12:16:17,338 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/B in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:17,338 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/A in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:17,339 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/3d06c501cfd64c348fc0c17a1a23bac3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=36.8 K 2024-11-20T12:16:17,339 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=92.3 K 2024-11-20T12:16:17,339 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7] 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d06c501cfd64c348fc0c17a1a23bac3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732104973669 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63848bbc8c714281b9cc6731cbacfb6b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732104973669 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f72f667c09de484494b8875419051aea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732104973757 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd375a209d144a1a0c310d2d38a4da0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732104973757 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b52127585e0046d8a94261e2472d80e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732104974491 2024-11-20T12:16:17,339 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8926f555bbe1433597b4e10139d4aae7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732104974491 2024-11-20T12:16:17,350 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#B#compaction#373 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:17,350 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/fa591b8cae08482eb08b1d5f11534e9f is 50, key is test_row_0/B:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:17,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742272_1448 (size=13153) 2024-11-20T12:16:17,354 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:17,355 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112088a84ee1db584ab89dd7cd49cccb05a6_bdb95162a28369d7de140a1abe1b7035 store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:17,357 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/fa591b8cae08482eb08b1d5f11534e9f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/fa591b8cae08482eb08b1d5f11534e9f 2024-11-20T12:16:17,361 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/B of bdb95162a28369d7de140a1abe1b7035 into fa591b8cae08482eb08b1d5f11534e9f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:17,361 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:17,362 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/B, priority=13, startTime=1732104977337; duration=0sec 2024-11-20T12:16:17,362 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:17,362 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:B 2024-11-20T12:16:17,362 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:17,363 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:17,363 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bdb95162a28369d7de140a1abe1b7035/C is initiating minor compaction (all files) 2024-11-20T12:16:17,363 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bdb95162a28369d7de140a1abe1b7035/C in TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:17,363 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c061d7c62a6d4e7b91e58cec02408be3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp, totalSize=48.7 K 2024-11-20T12:16:17,363 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c061d7c62a6d4e7b91e58cec02408be3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732104972334 2024-11-20T12:16:17,363 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c135a5b6e074a43a570f0b0cf02fb6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732104973669 2024-11-20T12:16:17,363 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e7f62cf142484393e605990aea6025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732104973757 2024-11-20T12:16:17,364 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 525aba15670a41de90ccaf7b6a7f2793, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=371, earliestPutTs=1732104974491 2024-11-20T12:16:17,377 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#C#compaction#375 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:17,377 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/47e14d0f4e8d4fe3856590ac244bb2df is 50, key is test_row_0/C:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:17,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742273_1449 (size=13119) 2024-11-20T12:16:17,385 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112088a84ee1db584ab89dd7cd49cccb05a6_bdb95162a28369d7de140a1abe1b7035, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:17,385 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112088a84ee1db584ab89dd7cd49cccb05a6_bdb95162a28369d7de140a1abe1b7035 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:17,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742274_1450 (size=4469) 2024-11-20T12:16:17,787 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/47e14d0f4e8d4fe3856590ac244bb2df as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/47e14d0f4e8d4fe3856590ac244bb2df 2024-11-20T12:16:17,789 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bdb95162a28369d7de140a1abe1b7035#A#compaction#374 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:17,790 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6f5a323ca5224ca881b3878c0d07a5f0 is 175, key is test_row_0/A:col10/1732104974491/Put/seqid=0 2024-11-20T12:16:17,793 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/C of bdb95162a28369d7de140a1abe1b7035 into 47e14d0f4e8d4fe3856590ac244bb2df(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:17,793 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:17,794 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/C, priority=12, startTime=1732104977338; duration=0sec 2024-11-20T12:16:17,794 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:17,794 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:C 2024-11-20T12:16:17,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742275_1451 (size=32107) 2024-11-20T12:16:17,808 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/6f5a323ca5224ca881b3878c0d07a5f0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6f5a323ca5224ca881b3878c0d07a5f0 2024-11-20T12:16:17,813 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bdb95162a28369d7de140a1abe1b7035/A of bdb95162a28369d7de140a1abe1b7035 into 6f5a323ca5224ca881b3878c0d07a5f0(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:17,813 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bdb95162a28369d7de140a1abe1b7035: 2024-11-20T12:16:17,813 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., storeName=bdb95162a28369d7de140a1abe1b7035/A, priority=13, startTime=1732104977337; duration=0sec 2024-11-20T12:16:17,813 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:17,813 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bdb95162a28369d7de140a1abe1b7035:A 2024-11-20T12:16:22,705 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:16:23,901 INFO [Thread-1583 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1583. 2024-11-20T12:16:23,901 DEBUG [Thread-1583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:23,902 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:23,902 DEBUG [Thread-1583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:23,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:23,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:16:23,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-20T12:16:23,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-20T12:16:23,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 849 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2547 rows 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 852 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2555 rows 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 858 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2574 rows 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 852 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2554 rows 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 855 2024-11-20T12:16:23,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2564 rows 2024-11-20T12:16:23,903 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:16:23,904 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:16:23,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:23,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:23,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:23,905 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:16:23,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:23,909 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104983909"}]},"ts":"1732104983909"} 2024-11-20T12:16:23,910 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:16:23,910 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:16:23,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:16:23,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, UNASSIGN}] 2024-11-20T12:16:23,913 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, UNASSIGN 2024-11-20T12:16:23,914 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:23,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, UNASSIGN because future has completed 2024-11-20T12:16:23,916 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:16:23,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:24,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:24,068 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:24,068 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:16:24,068 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing bdb95162a28369d7de140a1abe1b7035, disabling compactions & flushes 2024-11-20T12:16:24,068 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:24,068 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:24,068 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. after waiting 0 ms 2024-11-20T12:16:24,068 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:24,068 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing bdb95162a28369d7de140a1abe1b7035 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=A 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=B 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bdb95162a28369d7de140a1abe1b7035, store=C 2024-11-20T12:16:24,069 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:24,078 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207cd39c587117449bbf2e8e99cf222514_bdb95162a28369d7de140a1abe1b7035 is 50, key is test_row_0/A:col10/1732104975677/Put/seqid=0 2024-11-20T12:16:24,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742276_1452 (size=9914) 2024-11-20T12:16:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:24,483 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:24,492 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207cd39c587117449bbf2e8e99cf222514_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207cd39c587117449bbf2e8e99cf222514_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:24,493 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/157b7962e5f845f78f6accdbad103247, store: [table=TestAcidGuarantees family=A region=bdb95162a28369d7de140a1abe1b7035] 2024-11-20T12:16:24,494 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/157b7962e5f845f78f6accdbad103247 is 175, key is test_row_0/A:col10/1732104975677/Put/seqid=0 2024-11-20T12:16:24,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742277_1453 (size=22561) 2024-11-20T12:16:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:24,899 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/157b7962e5f845f78f6accdbad103247 2024-11-20T12:16:24,910 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/09519dcf2ecd43b58422ff08a6f1fc37 is 50, key is test_row_0/B:col10/1732104975677/Put/seqid=0 2024-11-20T12:16:24,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742278_1454 (size=9857) 2024-11-20T12:16:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:25,320 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/09519dcf2ecd43b58422ff08a6f1fc37 2024-11-20T12:16:25,334 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e2aea81685b840788b9db41de7d1f124 is 50, key is test_row_0/C:col10/1732104975677/Put/seqid=0 2024-11-20T12:16:25,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742279_1455 (size=9857) 2024-11-20T12:16:25,743 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e2aea81685b840788b9db41de7d1f124 2024-11-20T12:16:25,750 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/A/157b7962e5f845f78f6accdbad103247 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/157b7962e5f845f78f6accdbad103247 2024-11-20T12:16:25,753 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/157b7962e5f845f78f6accdbad103247, entries=100, sequenceid=379, filesize=22.0 K 2024-11-20T12:16:25,754 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/B/09519dcf2ecd43b58422ff08a6f1fc37 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/09519dcf2ecd43b58422ff08a6f1fc37 2024-11-20T12:16:25,757 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/09519dcf2ecd43b58422ff08a6f1fc37, entries=100, sequenceid=379, filesize=9.6 K 2024-11-20T12:16:25,758 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/.tmp/C/e2aea81685b840788b9db41de7d1f124 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e2aea81685b840788b9db41de7d1f124 2024-11-20T12:16:25,761 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e2aea81685b840788b9db41de7d1f124, entries=100, sequenceid=379, filesize=9.6 K 2024-11-20T12:16:25,762 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for bdb95162a28369d7de140a1abe1b7035 in 1694ms, sequenceid=379, compaction requested=false 2024-11-20T12:16:25,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7] to archive 2024-11-20T12:16:25,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:25,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cbde4c5bde8c41508216c8ef520ea185 2024-11-20T12:16:25,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/766f56e9de3d4538bb0a3b111cd24819 2024-11-20T12:16:25,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/585775ee23c647c7a093efacca43e2d4 2024-11-20T12:16:25,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/67097fca0d5f4db6baf92421eaeba9bb 2024-11-20T12:16:25,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0e76139abcc34e3aa3b2184233f97c18 2024-11-20T12:16:25,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6e6780bb694a474e885f095221380574 2024-11-20T12:16:25,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/b337849ee6d847e1914f263c3a1e68c8 2024-11-20T12:16:25,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/1a47303039fa467990d20bcac594a2f4 2024-11-20T12:16:25,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/c450941cdcbb415c9a5248c3ed6bef9d 2024-11-20T12:16:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/0d9f7a9993f94b99ac487c5226ccd779 2024-11-20T12:16:25,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/cc8d115410584acca37abde2b9ff9fe1 2024-11-20T12:16:25,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/ce629732919047b3a9fd86cef4caa9af 2024-11-20T12:16:25,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f4bf372d6a0a45758568c53de8f9a4bd 2024-11-20T12:16:25,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/e25067c63ff3464d9469c0ca693ce893 2024-11-20T12:16:25,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/be9d2c05d37e45e5ac63cfab3e0dd60b 2024-11-20T12:16:25,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/725e53b1c37448d7a1182438c2cd76b0 2024-11-20T12:16:25,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/3aeea1c7a7aa43ababa65eb085cc6544 2024-11-20T12:16:25,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/00de47c0d99d4459b786a558feb59f23 2024-11-20T12:16:25,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/f734881a8fb24761aa30b471b928a3bf 2024-11-20T12:16:25,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/d4990c05b9c24090b19f72ed02738135 2024-11-20T12:16:25,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/639eb955da7545ae9d0c2b33622245ac 2024-11-20T12:16:25,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/63848bbc8c714281b9cc6731cbacfb6b 2024-11-20T12:16:25,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/a85649f91f2c45c295bf3b2e77cecd67 2024-11-20T12:16:25,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/9bd375a209d144a1a0c310d2d38a4da0 2024-11-20T12:16:25,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/8926f555bbe1433597b4e10139d4aae7 2024-11-20T12:16:25,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/116280921b3045aea77f81bfe3438aea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/a8279a57ddae45c1bb3524db2ad2c0db, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b338d4d6063445279c2ea0d0237d509e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9514b0c80fa64754aad022806a7fa107, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9f5e8c6ae4244de78d51d1729928a12f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/3d06c501cfd64c348fc0c17a1a23bac3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6] to archive 2024-11-20T12:16:25,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:25,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/bca79b8584eb449690febd5e9a7e1a7e 2024-11-20T12:16:25,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/8b915b44dc5b4d668d0a3c1daa02bded 2024-11-20T12:16:25,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/116280921b3045aea77f81bfe3438aea to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/116280921b3045aea77f81bfe3438aea 2024-11-20T12:16:25,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5d872020c1bb4de0a80530265d351f9f 2024-11-20T12:16:25,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/407292de193944058be7ed62805e9698 2024-11-20T12:16:25,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/4759ab4ec54a40c6b896bf00d8a53f6b 2024-11-20T12:16:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f0e8ab9ed7264069b501ab6d5bfa30c3 2024-11-20T12:16:25,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/a8279a57ddae45c1bb3524db2ad2c0db to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/a8279a57ddae45c1bb3524db2ad2c0db 2024-11-20T12:16:25,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c8cbe6e8b3f4d238ceb4734a6105053 2024-11-20T12:16:25,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/483090cd36eb4b4e9af2fce8aead6b2b 2024-11-20T12:16:25,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b338d4d6063445279c2ea0d0237d509e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b338d4d6063445279c2ea0d0237d509e 2024-11-20T12:16:25,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5f05657522224b65a419da4fa626276c 2024-11-20T12:16:25,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ab092242c9bf435689faa1cd961aa68b 2024-11-20T12:16:25,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/6a268f43874d42cc9b6b416b6ba4ee97 2024-11-20T12:16:25,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/96c8fd3f8ebc48e69b884aa95ca48799 2024-11-20T12:16:25,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9514b0c80fa64754aad022806a7fa107 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9514b0c80fa64754aad022806a7fa107 2024-11-20T12:16:25,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f6c0a732acb64fb382edf3dc8c4649a6 2024-11-20T12:16:25,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e773f12a31ff4d189b5a84a95f137f30 2024-11-20T12:16:25,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9f5e8c6ae4244de78d51d1729928a12f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/9f5e8c6ae4244de78d51d1729928a12f 2024-11-20T12:16:25,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/ce1d713a5a2b4e5db621f49641a2fbdd 2024-11-20T12:16:25,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/e44790f8fba1412eb3e5433113522442 2024-11-20T12:16:25,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/3d06c501cfd64c348fc0c17a1a23bac3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/3d06c501cfd64c348fc0c17a1a23bac3 2024-11-20T12:16:25,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/5c0f575c254f43d3b288ad8404bfa2fe 2024-11-20T12:16:25,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/f72f667c09de484494b8875419051aea 2024-11-20T12:16:25,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/b52127585e0046d8a94261e2472d80e6 2024-11-20T12:16:25,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/dd1353769cef4598b18e23aec2325cb4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0b27fb5215154426829d98cc422ad1e4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/afd687a2ebc7486aa8fb09724e131a01, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a36d1650a5e94c51a666216fb4b4aef3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c061d7c62a6d4e7b91e58cec02408be3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793] to archive 2024-11-20T12:16:25,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:25,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/d14db7f4a2e9464080515d9f4d9f0a6d 2024-11-20T12:16:25,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/543ca0381de741e594e75544d31c0d19 2024-11-20T12:16:25,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/dd1353769cef4598b18e23aec2325cb4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/dd1353769cef4598b18e23aec2325cb4 2024-11-20T12:16:25,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/43b95fb95d334112b2b4ced8b1ce3a9f 2024-11-20T12:16:25,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/3bd133f7ed1f4131bd54966ad48c8353 2024-11-20T12:16:25,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a0945e6b915d4404955091f02979471c 2024-11-20T12:16:25,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/54b209e05d2e428dab85f023fea79215 2024-11-20T12:16:25,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0b27fb5215154426829d98cc422ad1e4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0b27fb5215154426829d98cc422ad1e4 2024-11-20T12:16:25,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1785b91d58d140849277462810708151 2024-11-20T12:16:25,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/41776f3494364adabd4ff10a975e7e15 2024-11-20T12:16:25,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/afd687a2ebc7486aa8fb09724e131a01 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/afd687a2ebc7486aa8fb09724e131a01 2024-11-20T12:16:25,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/4d600cd0d1eb435c8e5c3a04e8c0e693 2024-11-20T12:16:25,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/7f2c9a1d76b74d55a92e7f6e319f41ac 2024-11-20T12:16:25,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8676d66b46a2412cba801c5c4f1b071c 2024-11-20T12:16:25,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/5a32c7ba86344fcc8bd1806e1a548ca0 2024-11-20T12:16:25,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a36d1650a5e94c51a666216fb4b4aef3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/a36d1650a5e94c51a666216fb4b4aef3 2024-11-20T12:16:25,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/080e19de0c324e3ead28c838ed62d9b0 2024-11-20T12:16:25,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e7edb1b5b66b4aeba55b4fd625f26dc0 2024-11-20T12:16:25,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/0a80c185e36e4f0c82f47255150e268e 2024-11-20T12:16:25,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c061d7c62a6d4e7b91e58cec02408be3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c061d7c62a6d4e7b91e58cec02408be3 2024-11-20T12:16:25,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/1fafef4d2ab94d2ca9a787a83cd23dda 2024-11-20T12:16:25,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/8c135a5b6e074a43a570f0b0cf02fb6e 2024-11-20T12:16:25,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/c3e7f62cf142484393e605990aea6025 2024-11-20T12:16:25,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/525aba15670a41de90ccaf7b6a7f2793 2024-11-20T12:16:25,834 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-20T12:16:25,835 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. 2024-11-20T12:16:25,835 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for bdb95162a28369d7de140a1abe1b7035: Waiting for close lock at 1732104984068Running coprocessor pre-close hooks at 1732104984068Disabling compacts and flushes for region at 1732104984068Disabling writes for close at 1732104984068Obtaining lock to block concurrent updates at 1732104984068Preparing flush snapshotting stores in bdb95162a28369d7de140a1abe1b7035 at 1732104984068Finished memstore snapshotting TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1732104984069 (+1 ms)Flushing stores of TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035. at 1732104984070 (+1 ms)Flushing bdb95162a28369d7de140a1abe1b7035/A: creating writer at 1732104984070Flushing mob file bdb95162a28369d7de140a1abe1b7035/A: appending metadata at 1732104984078 (+8 ms)Flushing mob file bdb95162a28369d7de140a1abe1b7035/A: closing flushed file at 1732104984078Flushing bdb95162a28369d7de140a1abe1b7035/A: appending metadata at 1732104984493 (+415 ms)Flushing bdb95162a28369d7de140a1abe1b7035/A: closing flushed file at 1732104984494 (+1 ms)Flushing bdb95162a28369d7de140a1abe1b7035/B: creating writer at 1732104984907 (+413 ms)Flushing bdb95162a28369d7de140a1abe1b7035/B: appending metadata at 1732104984909 (+2 ms)Flushing bdb95162a28369d7de140a1abe1b7035/B: closing flushed file at 1732104984909Flushing bdb95162a28369d7de140a1abe1b7035/C: creating writer at 1732104985328 (+419 ms)Flushing bdb95162a28369d7de140a1abe1b7035/C: appending metadata at 1732104985333 (+5 ms)Flushing bdb95162a28369d7de140a1abe1b7035/C: closing flushed file at 1732104985333Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@111be2e5: reopening flushed file at 1732104985749 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a7f7293: reopening flushed file at 1732104985753 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39ac2a38: reopening flushed file at 1732104985757 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for bdb95162a28369d7de140a1abe1b7035 in 1694ms, sequenceid=379, compaction requested=false at 1732104985762 (+5 ms)Writing region close event to WAL at 1732104985828 (+66 ms)Running coprocessor post-close hooks at 1732104985835 (+7 ms)Closed at 1732104985835 2024-11-20T12:16:25,837 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:25,837 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=bdb95162a28369d7de140a1abe1b7035, regionState=CLOSED 2024-11-20T12:16:25,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:25,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T12:16:25,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure bdb95162a28369d7de140a1abe1b7035, server=843d83d51ba0,42825,1732104864746 in 1.9230 sec 2024-11-20T12:16:25,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-20T12:16:25,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bdb95162a28369d7de140a1abe1b7035, UNASSIGN in 1.9280 sec 2024-11-20T12:16:25,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T12:16:25,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9320 sec 2024-11-20T12:16:25,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104985846"}]},"ts":"1732104985846"} 2024-11-20T12:16:25,847 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:16:25,847 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:16:25,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9430 sec 2024-11-20T12:16:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T12:16:26,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:26,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:16:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,060 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T12:16:26,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,064 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,067 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits] 2024-11-20T12:16:26,071 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/157b7962e5f845f78f6accdbad103247 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/157b7962e5f845f78f6accdbad103247 2024-11-20T12:16:26,072 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6f5a323ca5224ca881b3878c0d07a5f0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/A/6f5a323ca5224ca881b3878c0d07a5f0 2024-11-20T12:16:26,075 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/09519dcf2ecd43b58422ff08a6f1fc37 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/09519dcf2ecd43b58422ff08a6f1fc37 2024-11-20T12:16:26,076 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/fa591b8cae08482eb08b1d5f11534e9f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/B/fa591b8cae08482eb08b1d5f11534e9f 2024-11-20T12:16:26,079 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/47e14d0f4e8d4fe3856590ac244bb2df to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/47e14d0f4e8d4fe3856590ac244bb2df 2024-11-20T12:16:26,081 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e2aea81685b840788b9db41de7d1f124 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/C/e2aea81685b840788b9db41de7d1f124 2024-11-20T12:16:26,084 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits/382.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035/recovered.edits/382.seqid 2024-11-20T12:16:26,085 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,085 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:16:26,086 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:16:26,086 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T12:16:26,091 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000739d287b1e403c8203583dffed7fa1_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112000739d287b1e403c8203583dffed7fa1_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,093 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b2d08d0d39648b3a5acdac75f0356a7_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b2d08d0d39648b3a5acdac75f0356a7_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,094 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201592b78f9c24471caab68538fcfb08c4_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201592b78f9c24471caab68538fcfb08c4_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,096 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202ec22aa1660c44b59becc79b5a063562_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202ec22aa1660c44b59becc79b5a063562_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,098 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d64a2c23a954b78a863543927fd2166_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205d64a2c23a954b78a863543927fd2166_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,099 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061567032425d4125b1634704ec017fd2_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112061567032425d4125b1634704ec017fd2_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,101 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120652f0941e5bc4f70a44337302bec4369_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120652f0941e5bc4f70a44337302bec4369_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,102 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120656a9bfc4b514a20a2a2a4e33d615456_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120656a9bfc4b514a20a2a2a4e33d615456_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,102 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c1489df05894eb3b27d68255e80c77a_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206c1489df05894eb3b27d68255e80c77a_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,103 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076009d71bb9749059511fd7d929865fb_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112076009d71bb9749059511fd7d929865fb_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,104 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207cd39c587117449bbf2e8e99cf222514_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207cd39c587117449bbf2e8e99cf222514_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,105 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112080ca1e7ab2c840219d19fade0ffd5d62_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112080ca1e7ab2c840219d19fade0ffd5d62_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,106 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112094533680854e46acaf1337159e3db9b2_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112094533680854e46acaf1337159e3db9b2_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,107 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee77018c66a4a51bcb0e4cae8ea0b61_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee77018c66a4a51bcb0e4cae8ea0b61_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,108 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3b612fbcf34d21bc7560d090b2e880_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ba3b612fbcf34d21bc7560d090b2e880_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,109 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bad36962b15c4d8a8c7a3f792e938316_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bad36962b15c4d8a8c7a3f792e938316_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,110 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5251c86ed44b5aab46558360eff56c_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120be5251c86ed44b5aab46558360eff56c_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,111 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d46b73a399794d1ab351501f6d94ec43_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d46b73a399794d1ab351501f6d94ec43_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,112 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d5c3e08fdee64db79dc5707722afca36_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d5c3e08fdee64db79dc5707722afca36_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,113 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f9be3a5b54474707b7808766054ca882_bdb95162a28369d7de140a1abe1b7035 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f9be3a5b54474707b7808766054ca882_bdb95162a28369d7de140a1abe1b7035 2024-11-20T12:16:26,113 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:16:26,115 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,118 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:16:26,121 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:16:26,122 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,122 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:16:26,122 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732104986122"}]},"ts":"9223372036854775807"} 2024-11-20T12:16:26,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:16:26,124 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bdb95162a28369d7de140a1abe1b7035, NAME => 'TestAcidGuarantees,,1732104952931.bdb95162a28369d7de140a1abe1b7035.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:16:26,124 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:16:26,124 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732104986124"}]},"ts":"9223372036854775807"} 2024-11-20T12:16:26,125 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:16:26,126 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-20T12:16:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T12:16:26,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:16:26,172 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:26,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 250), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 505), ProcessCount=11 (was 11), AvailableMemoryMB=2581 (was 3217) 2024-11-20T12:16:26,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=2581 2024-11-20T12:16:26,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:16:26,196 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:16:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:26,198 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:16:26,199 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:26,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-20T12:16:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:26,200 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:16:26,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742280_1456 (size=1061) 2024-11-20T12:16:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:26,607 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:16:26,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742281_1457 (size=53) 2024-11-20T12:16:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bf1698cac1d4eef2a4574349a5374fe2, disabling compactions & flushes 2024-11-20T12:16:27,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. after waiting 0 ms 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,015 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,015 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bf1698cac1d4eef2a4574349a5374fe2: Waiting for close lock at 1732104987015Disabling compacts and flushes for region at 1732104987015Disabling writes for close at 1732104987015Writing region close event to WAL at 1732104987015Closed at 1732104987015 2024-11-20T12:16:27,018 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:16:27,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732104987018"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104987018"}]},"ts":"1732104987018"} 2024-11-20T12:16:27,022 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:16:27,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:16:27,024 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104987024"}]},"ts":"1732104987024"} 2024-11-20T12:16:27,025 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:16:27,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, ASSIGN}] 2024-11-20T12:16:27,026 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, ASSIGN 2024-11-20T12:16:27,027 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:16:27,178 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=bf1698cac1d4eef2a4574349a5374fe2, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:27,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, ASSIGN because future has completed 2024-11-20T12:16:27,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:27,334 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,335 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:16:27,335 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,335 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:16:27,335 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,335 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:27,337 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,338 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:27,338 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1698cac1d4eef2a4574349a5374fe2 columnFamilyName A 2024-11-20T12:16:27,338 DEBUG [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:27,339 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(327): Store=bf1698cac1d4eef2a4574349a5374fe2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:27,339 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,340 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:27,340 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1698cac1d4eef2a4574349a5374fe2 columnFamilyName B 2024-11-20T12:16:27,340 DEBUG [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:27,341 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(327): Store=bf1698cac1d4eef2a4574349a5374fe2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:27,341 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,342 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:27,342 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bf1698cac1d4eef2a4574349a5374fe2 columnFamilyName C 2024-11-20T12:16:27,342 DEBUG [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:27,342 INFO [StoreOpener-bf1698cac1d4eef2a4574349a5374fe2-1 {}] regionserver.HStore(327): Store=bf1698cac1d4eef2a4574349a5374fe2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:27,342 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,343 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,343 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,344 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,345 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,345 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,345 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:16:27,347 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,349 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:16:27,350 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened bf1698cac1d4eef2a4574349a5374fe2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66933343, jitterRate=-0.0026154667139053345}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:16:27,350 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:27,351 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for bf1698cac1d4eef2a4574349a5374fe2: Running coprocessor pre-open hook at 1732104987335Writing region info on filesystem at 1732104987335Initializing all the Stores at 1732104987336 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104987336Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104987336Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104987336Cleaning up temporary data from old regions at 1732104987345 (+9 ms)Running coprocessor post-open hooks at 1732104987350 (+5 ms)Region opened successfully at 1732104987351 (+1 ms) 2024-11-20T12:16:27,352 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., pid=132, masterSystemTime=1732104987332 2024-11-20T12:16:27,354 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,354 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:27,354 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=bf1698cac1d4eef2a4574349a5374fe2, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:27,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:27,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T12:16:27,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 in 177 msec 2024-11-20T12:16:27,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-20T12:16:27,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, ASSIGN in 334 msec 2024-11-20T12:16:27,361 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:16:27,362 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104987361"}]},"ts":"1732104987361"} 2024-11-20T12:16:27,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:16:27,363 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:16:27,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-20T12:16:28,213 WARN [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 1, running: 1 2024-11-20T12:16:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-20T12:16:28,347 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:28,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@845334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216e07de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,354 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,354 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e0b0613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,355 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,358 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:16:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:16:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4dcff2bf 2024-11-20T12:16:28,359 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:16:28,361 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:16:28,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a6d3d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,363 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d4b7b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,366 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e18bc14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3734923a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,367 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5ea3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,369 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebfa20a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f704f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,371 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25361e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,373 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765a25a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,375 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b7894c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,375 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,375 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,376 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c2bdab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,378 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd50391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1447b9d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,380 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,380 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34400652, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec15334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f0c5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,383 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6af18e5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,384 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fdaefe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f86346d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,386 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcebe6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5afe0502, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,388 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b469189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,388 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,388 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,389 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdabe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,389 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b36dc84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6240d801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,391 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87b34dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,392 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c935e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2d9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,393 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:28,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:28,393 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58605adf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:28,394 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:28,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b67a4a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:28,395 DEBUG [Thread-2061 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,395 DEBUG [Thread-2057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,395 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,395 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,395 DEBUG [Thread-2059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,396 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,396 DEBUG [Thread-2048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,396 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,397 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,397 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:28,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:28,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:28,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T12:16:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,400 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:28,401 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:28,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:28,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,406 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2] 2024-11-20T12:16:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:28,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:28,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7846e0ec4bed40189775e124a002f443 is 50, key is test_row_0/A:col10/1732104988404/Put/seqid=0 2024-11-20T12:16:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742282_1458 (size=14341) 2024-11-20T12:16:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105048451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105048452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732104988445, max timestamp=1732104988470Num uniques 859; 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732104988445, max timestamp=1732104988470Num uniques 859; 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1732104988445, max timestamp=1732104988470Num uniques 859; 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:28,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:28,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105048539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105048546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105048547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105048596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,606 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105048596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105048657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105048657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105048666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:28,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105048817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105048817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7846e0ec4bed40189775e124a002f443 2024-11-20T12:16:28,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:28,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:28,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:28,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/d180bf1e53af411f8b788e8305ebb32a is 50, key is test_row_0/B:col10/1732104988404/Put/seqid=0 2024-11-20T12:16:28,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742283_1459 (size=12001) 2024-11-20T12:16:28,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105048876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105048876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:28,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105048876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:28,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:29,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105049136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105049136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105049196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105049196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105049196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,276 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/d180bf1e53af411f8b788e8305ebb32a 2024-11-20T12:16:29,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/aebea02ece9f4535b71c1401092740f8 is 50, key is test_row_0/C:col10/1732104988404/Put/seqid=0 2024-11-20T12:16:29,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742284_1460 (size=12001) 2024-11-20T12:16:29,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:29,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105049657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105049657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/aebea02ece9f4535b71c1401092740f8 2024-11-20T12:16:29,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7846e0ec4bed40189775e124a002f443 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443 2024-11-20T12:16:29,703 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443, entries=200, sequenceid=13, filesize=14.0 K 2024-11-20T12:16:29,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/d180bf1e53af411f8b788e8305ebb32a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a 2024-11-20T12:16:29,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T12:16:29,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/aebea02ece9f4535b71c1401092740f8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8 2024-11-20T12:16:29,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-20T12:16:29,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for bf1698cac1d4eef2a4574349a5374fe2 in 1295ms, sequenceid=13, compaction requested=false 2024-11-20T12:16:29,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:29,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:29,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:29,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/f40d466f67074c14b79b0dc5b2a876b6 is 50, key is test_row_0/A:col10/1732104988546/Put/seqid=0 2024-11-20T12:16:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742285_1461 (size=14341) 2024-11-20T12:16:29,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/f40d466f67074c14b79b0dc5b2a876b6 2024-11-20T12:16:29,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/cf1f9a0017ee445b978b2bf9e2252a9a is 50, key is test_row_0/B:col10/1732104988546/Put/seqid=0 2024-11-20T12:16:29,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105049750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742286_1462 (size=12001) 2024-11-20T12:16:29,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/cf1f9a0017ee445b978b2bf9e2252a9a 2024-11-20T12:16:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105049754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105049755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/0c68a8928cfe4252a40508fd79ab25bf is 50, key is test_row_0/C:col10/1732104988546/Put/seqid=0 2024-11-20T12:16:29,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742287_1463 (size=12001) 2024-11-20T12:16:29,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105049876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105049876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105049876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:29,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:29,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:30,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:30,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:30,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:30,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105050097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105050097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105050097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/0c68a8928cfe4252a40508fd79ab25bf 2024-11-20T12:16:30,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/f40d466f67074c14b79b0dc5b2a876b6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6 2024-11-20T12:16:30,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6, entries=200, sequenceid=48, filesize=14.0 K 2024-11-20T12:16:30,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/cf1f9a0017ee445b978b2bf9e2252a9a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a 2024-11-20T12:16:30,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T12:16:30,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/0c68a8928cfe4252a40508fd79ab25bf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf 2024-11-20T12:16:30,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T12:16:30,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bf1698cac1d4eef2a4574349a5374fe2 in 481ms, sequenceid=48, compaction requested=false 2024-11-20T12:16:30,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:30,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T12:16:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:30,238 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:30,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/efd659fcec0c4963a1620ec74d06b08e is 50, key is test_row_0/A:col10/1732104989753/Put/seqid=0 2024-11-20T12:16:30,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742288_1464 (size=12001) 2024-11-20T12:16:30,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104990447, max timestamp=1732104990466Num uniques 859; 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104990447, max timestamp=1732104990466Num uniques 859; 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732104990447, max timestamp=1732104990466Num uniques 859; 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:30,469 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-20T12:16:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105050510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105050512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105050522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:30,650 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/efd659fcec0c4963a1620ec74d06b08e 2024-11-20T12:16:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105050637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105050637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105050637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,653 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/4199f37cdc0341f6a2edb8f5ff42e923 is 50, key is test_row_0/B:col10/1732104989753/Put/seqid=0 2024-11-20T12:16:30,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742289_1465 (size=12001) 2024-11-20T12:16:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105050676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105050676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,688 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:30,689 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105050857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105050857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105050857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:30,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,060 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/4199f37cdc0341f6a2edb8f5ff42e923 2024-11-20T12:16:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4498abb271304bfdacb7d7e5796da03c is 50, key is test_row_0/C:col10/1732104989753/Put/seqid=0 2024-11-20T12:16:31,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742290_1466 (size=12001) 2024-11-20T12:16:31,070 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4498abb271304bfdacb7d7e5796da03c 2024-11-20T12:16:31,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/efd659fcec0c4963a1620ec74d06b08e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e 2024-11-20T12:16:31,076 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T12:16:31,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/4199f37cdc0341f6a2edb8f5ff42e923 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923 2024-11-20T12:16:31,079 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T12:16:31,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4498abb271304bfdacb7d7e5796da03c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c 2024-11-20T12:16:31,082 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c, entries=150, sequenceid=59, filesize=11.7 K 2024-11-20T12:16:31,083 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for bf1698cac1d4eef2a4574349a5374fe2 in 845ms, sequenceid=59, compaction requested=true 2024-11-20T12:16:31,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:31,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:31,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T12:16:31,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T12:16:31,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T12:16:31,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6820 sec 2024-11-20T12:16:31,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.6880 sec 2024-11-20T12:16:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:31,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:31,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:31,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b5c5afd14bd3475f8433b8207a614272 is 50, key is test_row_0/A:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:31,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742291_1467 (size=14341) 2024-11-20T12:16:31,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105051203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,215 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105051204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,216 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105051212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105051327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105051327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105051327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,364 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:16:31,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105051547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105051547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105051547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b5c5afd14bd3475f8433b8207a614272 2024-11-20T12:16:31,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/c9dac18523fb450eadf34c7e12d7f49a is 50, key is test_row_0/B:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:31,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742292_1468 (size=12001) 2024-11-20T12:16:31,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105051867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105051867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105051867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:31,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:31,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/c9dac18523fb450eadf34c7e12d7f49a 2024-11-20T12:16:32,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8de86bd753c3452793e78e7508499ecf is 50, key is test_row_0/C:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:32,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742293_1469 (size=12001) 2024-11-20T12:16:32,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105052376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105052376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105052376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8de86bd753c3452793e78e7508499ecf 2024-11-20T12:16:32,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b5c5afd14bd3475f8433b8207a614272 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272 2024-11-20T12:16:32,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272, entries=200, sequenceid=93, filesize=14.0 K 2024-11-20T12:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/c9dac18523fb450eadf34c7e12d7f49a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a 2024-11-20T12:16:32,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T12:16:32,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8de86bd753c3452793e78e7508499ecf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf 2024-11-20T12:16:32,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf, entries=150, sequenceid=93, filesize=11.7 K 2024-11-20T12:16:32,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for bf1698cac1d4eef2a4574349a5374fe2 in 1243ms, sequenceid=93, compaction requested=true 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:32,425 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:32,425 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:32,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:32,426 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,426 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,426 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=53.7 K 2024-11-20T12:16:32,426 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=46.9 K 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d180bf1e53af411f8b788e8305ebb32a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104988404 2024-11-20T12:16:32,426 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7846e0ec4bed40189775e124a002f443, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104988404 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40d466f67074c14b79b0dc5b2a876b6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104988536 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1f9a0017ee445b978b2bf9e2252a9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104988544 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4199f37cdc0341f6a2edb8f5ff42e923, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732104989746 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd659fcec0c4963a1620ec74d06b08e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732104989746 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5c5afd14bd3475f8433b8207a614272, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:32,427 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c9dac18523fb450eadf34c7e12d7f49a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:32,435 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#392 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:32,435 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/5478c9cf01344680872a1c1ebd0cd04e is 50, key is test_row_0/A:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:32,438 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#391 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:32,438 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/adafd6006c6c461a94180b3e13ef223b is 50, key is test_row_0/B:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:32,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742294_1470 (size=12139) 2024-11-20T12:16:32,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742295_1471 (size=12139) 2024-11-20T12:16:32,457 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/adafd6006c6c461a94180b3e13ef223b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/adafd6006c6c461a94180b3e13ef223b 2024-11-20T12:16:32,461 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into adafd6006c6c461a94180b3e13ef223b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:32,461 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:32,461 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=12, startTime=1732104992425; duration=0sec 2024-11-20T12:16:32,461 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:32,461 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:32,461 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:32,462 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:32,462 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:32,462 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,462 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=46.9 K 2024-11-20T12:16:32,462 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting aebea02ece9f4535b71c1401092740f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732104988404 2024-11-20T12:16:32,462 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c68a8928cfe4252a40508fd79ab25bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732104988544 2024-11-20T12:16:32,462 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4498abb271304bfdacb7d7e5796da03c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1732104989746 2024-11-20T12:16:32,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8de86bd753c3452793e78e7508499ecf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:32,469 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#393 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:32,470 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/26eec63e8ee54d0e8b92a671e32318ad is 50, key is test_row_0/C:col10/1732104990496/Put/seqid=0 2024-11-20T12:16:32,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742296_1472 (size=12139) 2024-11-20T12:16:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T12:16:32,557 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:32,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T12:16:32,558 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:32,559 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:32,559 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:32,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:32,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:32,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7d11f0ecb6f5407dabc619d0d4a58556 is 50, key is test_row_0/A:col10/1732104991183/Put/seqid=0 2024-11-20T12:16:32,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742297_1473 (size=14341) 2024-11-20T12:16:32,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104992749, max timestamp=1732104992776Num uniques 859; 2024-11-20T12:16:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104992749, max timestamp=1732104992776Num uniques 859; 2024-11-20T12:16:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104992749, max timestamp=1732104992776Num uniques 859; 2024-11-20T12:16:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:32,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:32,853 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/5478c9cf01344680872a1c1ebd0cd04e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/5478c9cf01344680872a1c1ebd0cd04e 2024-11-20T12:16:32,856 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 5478c9cf01344680872a1c1ebd0cd04e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:32,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:32,856 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=12, startTime=1732104992425; duration=0sec 2024-11-20T12:16:32,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:32,856 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:32,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:32,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:32,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105052862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105052868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:32,883 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/26eec63e8ee54d0e8b92a671e32318ad as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/26eec63e8ee54d0e8b92a671e32318ad 2024-11-20T12:16:32,886 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into 26eec63e8ee54d0e8b92a671e32318ad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:32,886 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:32,886 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=12, startTime=1732104992425; duration=0sec 2024-11-20T12:16:32,886 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:32,886 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:32,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105052977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:32,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105052989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:32,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7d11f0ecb6f5407dabc619d0d4a58556 2024-11-20T12:16:33,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9c1801b74cbf4237a52108a8bb20fa66 is 50, key is test_row_0/B:col10/1732104991183/Put/seqid=0 2024-11-20T12:16:33,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742298_1474 (size=12001) 2024-11-20T12:16:33,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:33,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:33,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105053197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105053206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:33,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105053397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105053397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105053397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,404 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,404 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:33,404 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:33,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:33,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:33,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105053517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9c1801b74cbf4237a52108a8bb20fa66 2024-11-20T12:16:33,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/d866f0b01d374bd28f0a1964f2b289f7 is 50, key is test_row_0/C:col10/1732104991183/Put/seqid=0 2024-11-20T12:16:33,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742299_1475 (size=12001) 2024-11-20T12:16:33,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/d866f0b01d374bd28f0a1964f2b289f7 2024-11-20T12:16:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:33,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105053526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/7d11f0ecb6f5407dabc619d0d4a58556 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556 2024-11-20T12:16:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:33,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556, entries=200, sequenceid=105, filesize=14.0 K 2024-11-20T12:16:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9c1801b74cbf4237a52108a8bb20fa66 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66 2024-11-20T12:16:33,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66, entries=150, sequenceid=105, filesize=11.7 K 2024-11-20T12:16:33,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/d866f0b01d374bd28f0a1964f2b289f7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7 2024-11-20T12:16:33,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7, entries=150, sequenceid=105, filesize=11.7 K 2024-11-20T12:16:33,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bf1698cac1d4eef2a4574349a5374fe2 in 845ms, sequenceid=105, compaction requested=false 2024-11-20T12:16:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:33,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T12:16:33,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:33,627 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:33,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e5c69c290bda45009e79bb904e17c1a9 is 50, key is test_row_0/A:col10/1732104992844/Put/seqid=0 2024-11-20T12:16:33,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742300_1476 (size=12151) 2024-11-20T12:16:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:34,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:34,035 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e5c69c290bda45009e79bb904e17c1a9 2024-11-20T12:16:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9acad4c7e1cc4434aecc6eadf4ae63d3 is 50, key is test_row_0/B:col10/1732104992844/Put/seqid=0 2024-11-20T12:16:34,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742301_1477 (size=12151) 2024-11-20T12:16:34,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105054061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105054067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105054177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105054187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105054386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105054407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,445 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9acad4c7e1cc4434aecc6eadf4ae63d3 2024-11-20T12:16:34,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/86c36b9e519e4263b0ec1aaa93827c09 is 50, key is test_row_0/C:col10/1732104992844/Put/seqid=0 2024-11-20T12:16:34,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742302_1478 (size=12151) 2024-11-20T12:16:34,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105054696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:34,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105054727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:34,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:34,859 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/86c36b9e519e4263b0ec1aaa93827c09 2024-11-20T12:16:34,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e5c69c290bda45009e79bb904e17c1a9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9 2024-11-20T12:16:34,865 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9, entries=150, sequenceid=142, filesize=11.9 K 2024-11-20T12:16:34,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9acad4c7e1cc4434aecc6eadf4ae63d3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3 2024-11-20T12:16:34,868 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3, entries=150, sequenceid=142, filesize=11.9 K 2024-11-20T12:16:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/86c36b9e519e4263b0ec1aaa93827c09 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09 2024-11-20T12:16:34,872 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09, entries=150, sequenceid=142, filesize=11.9 K 2024-11-20T12:16:34,873 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bf1698cac1d4eef2a4574349a5374fe2 in 1247ms, sequenceid=142, compaction requested=true 2024-11-20T12:16:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T12:16:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T12:16:34,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T12:16:34,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3150 sec 2024-11-20T12:16:34,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3190 sec 2024-11-20T12:16:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:35,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:16:35,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:35,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:35,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:35,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:35,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:35,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:35,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/2e08f5305aa640a7b639b3a1204defe9 is 50, key is test_row_0/A:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742303_1479 (size=14541) 2024-11-20T12:16:35,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/2e08f5305aa640a7b639b3a1204defe9 2024-11-20T12:16:35,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 is 50, key is test_row_0/B:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742304_1480 (size=12151) 2024-11-20T12:16:35,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105055293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105055300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105055407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105055417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105055417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,426 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105055426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105055426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,434 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:35,434 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105055627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 2024-11-20T12:16:35,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105055636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/7a47c400775f490f9ceb4fb1cc4fb4d4 is 50, key is test_row_0/C:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742305_1481 (size=12151) 2024-11-20T12:16:35,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/7a47c400775f490f9ceb4fb1cc4fb4d4 2024-11-20T12:16:35,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/2e08f5305aa640a7b639b3a1204defe9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9 2024-11-20T12:16:35,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9, entries=200, sequenceid=154, filesize=14.2 K 2024-11-20T12:16:35,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 2024-11-20T12:16:35,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T12:16:35,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/7a47c400775f490f9ceb4fb1cc4fb4d4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4 2024-11-20T12:16:35,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4, entries=150, sequenceid=154, filesize=11.9 K 2024-11-20T12:16:35,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bf1698cac1d4eef2a4574349a5374fe2 in 443ms, sequenceid=154, compaction requested=true 2024-11-20T12:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:35,665 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:35,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:35,665 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:35,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:35,666 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:35,666 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:35,666 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:35,666 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/5478c9cf01344680872a1c1ebd0cd04e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=51.9 K 2024-11-20T12:16:35,666 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5478c9cf01344680872a1c1ebd0cd04e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d11f0ecb6f5407dabc619d0d4a58556, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732104991183 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:35,667 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5c69c290bda45009e79bb904e17c1a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732104992844 2024-11-20T12:16:35,667 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/adafd6006c6c461a94180b3e13ef223b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=47.3 K 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e08f5305aa640a7b639b3a1204defe9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:35,667 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting adafd6006c6c461a94180b3e13ef223b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:35,668 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c1801b74cbf4237a52108a8bb20fa66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732104991183 2024-11-20T12:16:35,668 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9acad4c7e1cc4434aecc6eadf4ae63d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732104992844 2024-11-20T12:16:35,668 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcb5045a6f7412ab5c8c3f47c20d8e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:35,674 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:35,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/fd77d5b0f5df4f2b93b5118985156f49 is 50, key is test_row_0/B:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,676 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:35,677 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/59179126537e4d3089f0af54c2bf6787 is 50, key is test_row_0/A:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742306_1482 (size=12425) 2024-11-20T12:16:35,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742307_1483 (size=12425) 2024-11-20T12:16:35,685 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/fd77d5b0f5df4f2b93b5118985156f49 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fd77d5b0f5df4f2b93b5118985156f49 2024-11-20T12:16:35,688 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into fd77d5b0f5df4f2b93b5118985156f49(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:35,688 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:35,688 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=12, startTime=1732104995665; duration=0sec 2024-11-20T12:16:35,688 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:35,688 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:35,688 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:35,689 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:35,689 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:35,689 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:35,689 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/26eec63e8ee54d0e8b92a671e32318ad, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=47.3 K 2024-11-20T12:16:35,690 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 26eec63e8ee54d0e8b92a671e32318ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732104990496 2024-11-20T12:16:35,690 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d866f0b01d374bd28f0a1964f2b289f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732104991183 2024-11-20T12:16:35,690 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c36b9e519e4263b0ec1aaa93827c09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1732104992844 2024-11-20T12:16:35,690 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a47c400775f490f9ceb4fb1cc4fb4d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:35,697 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:35,698 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/9afcf333c16a4923bab61fe20451aa00 is 50, key is test_row_0/C:col10/1732104994066/Put/seqid=0 2024-11-20T12:16:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742308_1484 (size=12425) 2024-11-20T12:16:35,706 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/9afcf333c16a4923bab61fe20451aa00 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9afcf333c16a4923bab61fe20451aa00 2024-11-20T12:16:35,709 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into 9afcf333c16a4923bab61fe20451aa00(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:35,710 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:35,710 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=12, startTime=1732104995666; duration=0sec 2024-11-20T12:16:35,710 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:35,710 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:35,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:35,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:35,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:35,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3d057461b83e491d98c9b2f6bff910ea is 50, key is test_row_0/A:col10/1732104995299/Put/seqid=0 2024-11-20T12:16:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742309_1485 (size=14541) 2024-11-20T12:16:35,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105055986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105055987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,087 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/59179126537e4d3089f0af54c2bf6787 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/59179126537e4d3089f0af54c2bf6787 2024-11-20T12:16:36,090 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 59179126537e4d3089f0af54c2bf6787(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:36,090 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:36,090 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=12, startTime=1732104995665; duration=0sec 2024-11-20T12:16:36,090 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:36,090 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105056097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105056107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105056306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105056317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3d057461b83e491d98c9b2f6bff910ea 2024-11-20T12:16:36,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0f627c9c2e034ab4ba7c81b32fc3cc80 is 50, key is test_row_0/B:col10/1732104995299/Put/seqid=0 2024-11-20T12:16:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742310_1486 (size=12151) 2024-11-20T12:16:36,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0f627c9c2e034ab4ba7c81b32fc3cc80 2024-11-20T12:16:36,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/9746dcee73d74824bc4a111ab835dc66 is 50, key is test_row_0/C:col10/1732104995299/Put/seqid=0 2024-11-20T12:16:36,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742311_1487 (size=12151) 2024-11-20T12:16:36,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105056617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105056637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T12:16:36,717 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:36,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-20T12:16:36,719 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T12:16:36,720 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:36,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:36,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/9746dcee73d74824bc4a111ab835dc66 2024-11-20T12:16:36,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3d057461b83e491d98c9b2f6bff910ea as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea 2024-11-20T12:16:36,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea, entries=200, sequenceid=177, filesize=14.2 K 2024-11-20T12:16:36,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0f627c9c2e034ab4ba7c81b32fc3cc80 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80 2024-11-20T12:16:36,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T12:16:36,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/9746dcee73d74824bc4a111ab835dc66 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66 2024-11-20T12:16:36,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T12:16:36,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for bf1698cac1d4eef2a4574349a5374fe2 in 844ms, sequenceid=177, compaction requested=false 2024-11-20T12:16:36,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T12:16:36,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:36,874 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:36,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3b18fcd2ce7746f394e6d22e45550388 is 50, key is test_row_0/A:col10/1732104995986/Put/seqid=0 2024-11-20T12:16:36,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742312_1488 (size=12151) 2024-11-20T12:16:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T12:16:37,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:37,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104997207, max timestamp=1732104997223Num uniques 859; 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104997207, max timestamp=1732104997223Num uniques 859; 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104997207, max timestamp=1732104997223Num uniques 859; 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:37,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:37,280 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3b18fcd2ce7746f394e6d22e45550388 2024-11-20T12:16:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e4fabf38dfa34038af88b185214a2047 is 50, key is test_row_0/B:col10/1732104995986/Put/seqid=0 2024-11-20T12:16:37,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742313_1489 (size=12151) 2024-11-20T12:16:37,292 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e4fabf38dfa34038af88b185214a2047 2024-11-20T12:16:37,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/3d9e7b519074454480f8863c475a44b3 is 50, key is test_row_0/C:col10/1732104995986/Put/seqid=0 2024-11-20T12:16:37,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742314_1490 (size=12151) 2024-11-20T12:16:37,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105057296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105057309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T12:16:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105057417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105057417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105057626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105057627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:37,703 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/3d9e7b519074454480f8863c475a44b3 2024-11-20T12:16:37,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/3b18fcd2ce7746f394e6d22e45550388 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388 2024-11-20T12:16:37,709 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:16:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e4fabf38dfa34038af88b185214a2047 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047 2024-11-20T12:16:37,715 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:16:37,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/3d9e7b519074454480f8863c475a44b3 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3 2024-11-20T12:16:37,718 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3, entries=150, sequenceid=189, filesize=11.9 K 2024-11-20T12:16:37,719 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bf1698cac1d4eef2a4574349a5374fe2 in 845ms, sequenceid=189, compaction requested=true 2024-11-20T12:16:37,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:37,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:37,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-20T12:16:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-20T12:16:37,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T12:16:37,722 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0010 sec 2024-11-20T12:16:37,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0040 sec 2024-11-20T12:16:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T12:16:37,856 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:37,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-20T12:16:37,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:37,858 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:37,859 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:37,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:37,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:37,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/4bf70b195a044bef95d5d30657320f24 is 50, key is test_row_0/A:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:37,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742315_1491 (size=14541) 2024-11-20T12:16:37,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/4bf70b195a044bef95d5d30657320f24 2024-11-20T12:16:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:37,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3354a9cdfe604fcfb1535c18289cadc2 is 50, key is test_row_0/B:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:37,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742316_1492 (size=12151) 2024-11-20T12:16:37,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105057983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:37,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105057989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105058107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105058107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:38,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105058327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105058327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3354a9cdfe604fcfb1535c18289cadc2 2024-11-20T12:16:38,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/ceca058a765d48e390ce4539c9874753 is 50, key is test_row_0/C:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:38,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742317_1493 (size=12151) 2024-11-20T12:16:38,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:38,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105058646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105058647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:38,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:38,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/ceca058a765d48e390ce4539c9874753 2024-11-20T12:16:38,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/4bf70b195a044bef95d5d30657320f24 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24 2024-11-20T12:16:38,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24, entries=200, sequenceid=225, filesize=14.2 K 2024-11-20T12:16:38,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/3354a9cdfe604fcfb1535c18289cadc2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2 2024-11-20T12:16:38,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:16:38,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/ceca058a765d48e390ce4539c9874753 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753 2024-11-20T12:16:38,796 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753, entries=150, sequenceid=225, filesize=11.9 K 2024-11-20T12:16:38,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bf1698cac1d4eef2a4574349a5374fe2 in 843ms, sequenceid=225, compaction requested=true 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:38,797 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:38,797 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:38,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:38,798 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:38,798 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:38,798 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,798 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fd77d5b0f5df4f2b93b5118985156f49, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=47.7 K 2024-11-20T12:16:38,798 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting fd77d5b0f5df4f2b93b5118985156f49, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:38,798 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:38,799 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:38,799 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,799 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/59179126537e4d3089f0af54c2bf6787, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=52.4 K 2024-11-20T12:16:38,799 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f627c9c2e034ab4ba7c81b32fc3cc80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732104995278 2024-11-20T12:16:38,799 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59179126537e4d3089f0af54c2bf6787, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:38,799 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d057461b83e491d98c9b2f6bff910ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732104995278 2024-11-20T12:16:38,799 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e4fabf38dfa34038af88b185214a2047, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104995984 2024-11-20T12:16:38,800 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b18fcd2ce7746f394e6d22e45550388, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104995984 2024-11-20T12:16:38,800 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3354a9cdfe604fcfb1535c18289cadc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997289 2024-11-20T12:16:38,800 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bf70b195a044bef95d5d30657320f24, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997287 2024-11-20T12:16:38,809 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:38,809 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9520d63ef54545e0adb79d5248474054 is 50, key is test_row_0/B:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:38,812 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:38,813 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/cd03afcbdcc4457ca902ab0f063169e1 is 50, key is test_row_0/A:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:38,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742318_1494 (size=12561) 2024-11-20T12:16:38,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742319_1495 (size=12561) 2024-11-20T12:16:38,833 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/cd03afcbdcc4457ca902ab0f063169e1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/cd03afcbdcc4457ca902ab0f063169e1 2024-11-20T12:16:38,836 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into cd03afcbdcc4457ca902ab0f063169e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:38,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:38,836 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=12, startTime=1732104998797; duration=0sec 2024-11-20T12:16:38,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:38,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:38,836 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:38,837 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:38,837 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:38,837 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,837 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9afcf333c16a4923bab61fe20451aa00, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=47.7 K 2024-11-20T12:16:38,838 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9afcf333c16a4923bab61fe20451aa00, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732104994057 2024-11-20T12:16:38,838 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9746dcee73d74824bc4a111ab835dc66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732104995278 2024-11-20T12:16:38,838 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9e7b519074454480f8863c475a44b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732104995984 2024-11-20T12:16:38,838 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceca058a765d48e390ce4539c9874753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997289 2024-11-20T12:16:38,845 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#417 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:38,845 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f005fcdd198348e9829858879c1aa8e6 is 50, key is test_row_0/C:col10/1732104997307/Put/seqid=0 2024-11-20T12:16:38,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742320_1496 (size=12561) 2024-11-20T12:16:38,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-20T12:16:38,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:38,928 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:38,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:38,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/aa4e6403f9344a3a9b2917732a0ae098 is 50, key is test_row_0/A:col10/1732104997980/Put/seqid=0 2024-11-20T12:16:38,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742321_1497 (size=12151) 2024-11-20T12:16:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:39,227 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/9520d63ef54545e0adb79d5248474054 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9520d63ef54545e0adb79d5248474054 2024-11-20T12:16:39,230 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into 9520d63ef54545e0adb79d5248474054(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:39,230 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:39,230 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=12, startTime=1732104998797; duration=0sec 2024-11-20T12:16:39,230 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:39,230 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:39,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104999208, max timestamp=1732104999233Num uniques 859; 2024-11-20T12:16:39,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:39,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104999208, max timestamp=1732104999233Num uniques 859; 2024-11-20T12:16:39,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:39,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732104999208, max timestamp=1732104999233Num uniques 859; 2024-11-20T12:16:39,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:39,252 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f005fcdd198348e9829858879c1aa8e6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f005fcdd198348e9829858879c1aa8e6 2024-11-20T12:16:39,255 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into f005fcdd198348e9829858879c1aa8e6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:39,255 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:39,255 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=12, startTime=1732104998797; duration=0sec 2024-11-20T12:16:39,255 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:39,255 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105059303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105059311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,340 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/aa4e6403f9344a3a9b2917732a0ae098 2024-11-20T12:16:39,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0b9d5026213b4a109a5f2f7be736ca30 is 50, key is test_row_0/B:col10/1732104997980/Put/seqid=0 2024-11-20T12:16:39,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742322_1498 (size=12151) 2024-11-20T12:16:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105059426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105059427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43374 deadline: 1732105059437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,446 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43392 deadline: 1732105059446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,456 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_1' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43436 deadline: 1732105059477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,482 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in bf1698cac1d4eef2a4574349a5374fe2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105059637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105059637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,749 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0b9d5026213b4a109a5f2f7be736ca30 2024-11-20T12:16:39,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/53d6a9a2fde64d7381f2c53234d24229 is 50, key is test_row_0/C:col10/1732104997980/Put/seqid=0 2024-11-20T12:16:39,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742323_1499 (size=12151) 2024-11-20T12:16:39,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105059946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:39,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105059956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:40,162 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/53d6a9a2fde64d7381f2c53234d24229 2024-11-20T12:16:40,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/aa4e6403f9344a3a9b2917732a0ae098 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098 2024-11-20T12:16:40,169 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:16:40,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/0b9d5026213b4a109a5f2f7be736ca30 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30 2024-11-20T12:16:40,172 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:16:40,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/53d6a9a2fde64d7381f2c53234d24229 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229 2024-11-20T12:16:40,176 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229, entries=150, sequenceid=236, filesize=11.9 K 2024-11-20T12:16:40,176 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bf1698cac1d4eef2a4574349a5374fe2 in 1248ms, sequenceid=236, compaction requested=false 2024-11-20T12:16:40,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:40,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:40,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-20T12:16:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-20T12:16:40,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T12:16:40,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-20T12:16:40,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.3230 sec 2024-11-20T12:16:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:40,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-20T12:16:40,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:40,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:40,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e927fe29e45e4744bf2d96ff894f4f03 is 50, key is test_row_0/A:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:40,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742324_1500 (size=14741) 2024-11-20T12:16:40,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105060510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105060517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105060627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105060637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105060847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,855 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105060857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:40,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:40,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e927fe29e45e4744bf2d96ff894f4f03 2024-11-20T12:16:40,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/799f829d0ede440ca0574a326d77f546 is 50, key is test_row_0/B:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:40,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742325_1501 (size=12301) 2024-11-20T12:16:40,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/799f829d0ede440ca0574a326d77f546 2024-11-20T12:16:40,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f4d5991f406545ed9638ea20d8bc9f77 is 50, key is test_row_0/C:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742326_1502 (size=12301) 2024-11-20T12:16:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105061167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105061176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:41,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f4d5991f406545ed9638ea20d8bc9f77 2024-11-20T12:16:41,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/e927fe29e45e4744bf2d96ff894f4f03 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03 2024-11-20T12:16:41,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03, entries=200, sequenceid=275, filesize=14.4 K 2024-11-20T12:16:41,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/799f829d0ede440ca0574a326d77f546 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546 2024-11-20T12:16:41,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T12:16:41,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f4d5991f406545ed9638ea20d8bc9f77 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77 2024-11-20T12:16:41,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77, entries=150, sequenceid=275, filesize=12.0 K 2024-11-20T12:16:41,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for bf1698cac1d4eef2a4574349a5374fe2 in 863ms, sequenceid=275, compaction requested=true 2024-11-20T12:16:41,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:41,336 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:41,336 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:41,337 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39453 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:41,337 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:41,337 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:41,337 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/cd03afcbdcc4457ca902ab0f063169e1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=38.5 K 2024-11-20T12:16:41,337 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:41,337 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:41,337 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:41,337 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9520d63ef54545e0adb79d5248474054, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=36.1 K 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd03afcbdcc4457ca902ab0f063169e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997289 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9520d63ef54545e0adb79d5248474054, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997289 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa4e6403f9344a3a9b2917732a0ae098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104997970 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b9d5026213b4a109a5f2f7be736ca30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104997970 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting e927fe29e45e4744bf2d96ff894f4f03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:41,338 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 799f829d0ede440ca0574a326d77f546, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:41,345 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:41,345 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:41,346 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/2ee7718a2cc24b96b184bbfaeea700a1 is 50, key is test_row_0/B:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:41,346 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/4a91a5ae0f504f01b15d73b46918db59 is 50, key is test_row_0/A:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:41,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742327_1503 (size=12813) 2024-11-20T12:16:41,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742328_1504 (size=12813) 2024-11-20T12:16:41,371 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/2ee7718a2cc24b96b184bbfaeea700a1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2ee7718a2cc24b96b184bbfaeea700a1 2024-11-20T12:16:41,379 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/4a91a5ae0f504f01b15d73b46918db59 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4a91a5ae0f504f01b15d73b46918db59 2024-11-20T12:16:41,383 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into 2ee7718a2cc24b96b184bbfaeea700a1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:41,383 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:41,383 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=13, startTime=1732105001336; duration=0sec 2024-11-20T12:16:41,383 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:41,383 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:41,383 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:41,384 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37013 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:41,384 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:41,384 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:41,384 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f005fcdd198348e9829858879c1aa8e6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=36.1 K 2024-11-20T12:16:41,385 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f005fcdd198348e9829858879c1aa8e6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1732104997289 2024-11-20T12:16:41,385 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d6a9a2fde64d7381f2c53234d24229, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732104997970 2024-11-20T12:16:41,385 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d5991f406545ed9638ea20d8bc9f77, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:41,393 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 4a91a5ae0f504f01b15d73b46918db59(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:41,394 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:41,394 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=13, startTime=1732105001335; duration=0sec 2024-11-20T12:16:41,394 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:41,394 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:41,404 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#426 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:41,404 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f9a05bc85f0c410d8e5f5d237c25fb8b is 50, key is test_row_0/C:col10/1732104999310/Put/seqid=0 2024-11-20T12:16:41,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742329_1505 (size=12813) 2024-11-20T12:16:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:41,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:41,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:41,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b06bfa74f4144d719907d0f18a2f00e7 is 50, key is test_row_0/A:col10/1732105001690/Put/seqid=0 2024-11-20T12:16:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742330_1506 (size=14741) 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105001723, max timestamp=1732105001754Num uniques 859; 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105001723, max timestamp=1732105001754Num uniques 859; 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105001723, max timestamp=1732105001754Num uniques 859; 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:41,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105061808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,820 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:41,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105061820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:41,829 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f9a05bc85f0c410d8e5f5d237c25fb8b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f9a05bc85f0c410d8e5f5d237c25fb8b 2024-11-20T12:16:41,833 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into f9a05bc85f0c410d8e5f5d237c25fb8b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:41,833 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:41,833 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=13, startTime=1732105001336; duration=0sec 2024-11-20T12:16:41,833 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:41,833 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105061926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105061926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:41,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T12:16:42,016 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:42,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-20T12:16:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:42,018 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:42,019 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:42,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:42,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b06bfa74f4144d719907d0f18a2f00e7 2024-11-20T12:16:42,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/424e0056cae14936961dccee48b89497 is 50, key is test_row_0/B:col10/1732105001690/Put/seqid=0 2024-11-20T12:16:42,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742331_1507 (size=12301) 2024-11-20T12:16:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105062146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105062146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:42,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:42,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105062456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:42,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:42,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105062457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:42,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/424e0056cae14936961dccee48b89497 2024-11-20T12:16:42,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4c61d63ec8664181af9ea6a563fcb9e7 is 50, key is test_row_0/C:col10/1732105001690/Put/seqid=0 2024-11-20T12:16:42,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742332_1508 (size=12301) 2024-11-20T12:16:42,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:42,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4c61d63ec8664181af9ea6a563fcb9e7 2024-11-20T12:16:42,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/b06bfa74f4144d719907d0f18a2f00e7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7 2024-11-20T12:16:42,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:42,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:42,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7, entries=200, sequenceid=287, filesize=14.4 K 2024-11-20T12:16:42,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/424e0056cae14936961dccee48b89497 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497 2024-11-20T12:16:42,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T12:16:42,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/4c61d63ec8664181af9ea6a563fcb9e7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7 2024-11-20T12:16:42,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7, entries=150, sequenceid=287, filesize=12.0 K 2024-11-20T12:16:42,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bf1698cac1d4eef2a4574349a5374fe2 in 1251ms, sequenceid=287, compaction requested=false 2024-11-20T12:16:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:42,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-20T12:16:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:42,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T12:16:43,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/ea1e61cec45c40af9d03a7c2d7cc03f6 is 50, key is test_row_0/A:col10/1732105002990/Put/seqid=0 2024-11-20T12:16:43,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742333_1509 (size=17181) 2024-11-20T12:16:43,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/ea1e61cec45c40af9d03a7c2d7cc03f6 2024-11-20T12:16:43,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/fe61a02b44404d4f98181cb6abcfa6c4 is 50, key is test_row_0/B:col10/1732105002990/Put/seqid=0 2024-11-20T12:16:43,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742334_1510 (size=12301) 2024-11-20T12:16:43,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105063042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105063042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105063147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105063147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:43,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105063357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105063357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:43,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:43,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/fe61a02b44404d4f98181cb6abcfa6c4 2024-11-20T12:16:43,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f157dcbdcc0f41bbb7823282e40847ab is 50, key is test_row_0/C:col10/1732105002990/Put/seqid=0 2024-11-20T12:16:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742335_1511 (size=12301) 2024-11-20T12:16:43,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f157dcbdcc0f41bbb7823282e40847ab 2024-11-20T12:16:43,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/ea1e61cec45c40af9d03a7c2d7cc03f6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6 2024-11-20T12:16:43,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6, entries=250, sequenceid=326, filesize=16.8 K 2024-11-20T12:16:43,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/fe61a02b44404d4f98181cb6abcfa6c4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4 2024-11-20T12:16:43,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T12:16:43,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f157dcbdcc0f41bbb7823282e40847ab as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab 2024-11-20T12:16:43,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab, entries=150, sequenceid=326, filesize=12.0 K 2024-11-20T12:16:43,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=40.25 KB/41220 for bf1698cac1d4eef2a4574349a5374fe2 in 497ms, sequenceid=326, compaction requested=true 2024-11-20T12:16:43,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:43,492 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:43,492 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:43,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:43,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:43,493 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44735 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:43,493 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:43,493 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,494 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4a91a5ae0f504f01b15d73b46918db59, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=43.7 K 2024-11-20T12:16:43,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:43,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:43,494 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,494 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2ee7718a2cc24b96b184bbfaeea700a1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=36.5 K 2024-11-20T12:16:43,494 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a91a5ae0f504f01b15d73b46918db59, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:43,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee7718a2cc24b96b184bbfaeea700a1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:43,495 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06bfa74f4144d719907d0f18a2f00e7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732105000508 2024-11-20T12:16:43,495 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 424e0056cae14936961dccee48b89497, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732105000508 2024-11-20T12:16:43,496 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting fe61a02b44404d4f98181cb6abcfa6c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732105001807 2024-11-20T12:16:43,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea1e61cec45c40af9d03a7c2d7cc03f6, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732105001795 2024-11-20T12:16:43,506 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#433 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:43,506 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/8252cc0b60cb47228d7c4bc66f78ff32 is 50, key is test_row_0/B:col10/1732105002990/Put/seqid=0 2024-11-20T12:16:43,508 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:43,509 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/6b8d2a520df0453e864d98163ef3be5a is 50, key is test_row_0/A:col10/1732105002990/Put/seqid=0 2024-11-20T12:16:43,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742337_1513 (size=12915) 2024-11-20T12:16:43,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,545 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:43,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742336_1512 (size=12915) 2024-11-20T12:16:43,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/dcfeaaa0ea5d440b91327b0284c95e72 is 50, key is test_row_0/A:col10/1732105003041/Put/seqid=0 2024-11-20T12:16:43,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742338_1514 (size=12301) 2024-11-20T12:16:43,581 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/dcfeaaa0ea5d440b91327b0284c95e72 2024-11-20T12:16:43,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e35d724f1d724504b9da4f17c8994bdb is 50, key is test_row_0/B:col10/1732105003041/Put/seqid=0 2024-11-20T12:16:43,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742339_1515 (size=12301) 2024-11-20T12:16:43,602 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e35d724f1d724504b9da4f17c8994bdb 2024-11-20T12:16:43,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/cea4f89403cb4bafaf13189fb657a3ce is 50, key is test_row_0/C:col10/1732105003041/Put/seqid=0 2024-11-20T12:16:43,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742340_1516 (size=12301) 2024-11-20T12:16:43,617 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/cea4f89403cb4bafaf13189fb657a3ce 2024-11-20T12:16:43,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/dcfeaaa0ea5d440b91327b0284c95e72 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72 2024-11-20T12:16:43,623 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72, entries=150, sequenceid=335, filesize=12.0 K 2024-11-20T12:16:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e35d724f1d724504b9da4f17c8994bdb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb 2024-11-20T12:16:43,626 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb, entries=150, sequenceid=335, filesize=12.0 K 2024-11-20T12:16:43,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/cea4f89403cb4bafaf13189fb657a3ce as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce 2024-11-20T12:16:43,630 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce, entries=150, sequenceid=335, filesize=12.0 K 2024-11-20T12:16:43,631 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for bf1698cac1d4eef2a4574349a5374fe2 in 86ms, sequenceid=335, compaction requested=true 2024-11-20T12:16:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-20T12:16:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-20T12:16:43,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T12:16:43,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-20T12:16:43,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.6160 sec 2024-11-20T12:16:43,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:43,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:43,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:43,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/167ff699b7d04e9b81c18c37ab6ef4f5 is 50, key is test_row_0/A:col10/1732105003687/Put/seqid=0 2024-11-20T12:16:43,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742341_1517 (size=22065) 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105003744, max timestamp=1732105003764Num uniques 859; 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105003744, max timestamp=1732105003764Num uniques 859; 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105003744, max timestamp=1732105003764Num uniques 859; 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:43,767 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:43,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105063841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105063856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,934 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/6b8d2a520df0453e864d98163ef3be5a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/6b8d2a520df0453e864d98163ef3be5a 2024-11-20T12:16:43,938 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 6b8d2a520df0453e864d98163ef3be5a(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:43,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:43,938 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=13, startTime=1732105003492; duration=0sec 2024-11-20T12:16:43,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:43,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:43,938 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:43,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:43,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:43,939 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:43,939 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f9a05bc85f0c410d8e5f5d237c25fb8b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=48.6 K 2024-11-20T12:16:43,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9a05bc85f0c410d8e5f5d237c25fb8b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732104999296 2024-11-20T12:16:43,939 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c61d63ec8664181af9ea6a563fcb9e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732105000508 2024-11-20T12:16:43,940 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f157dcbdcc0f41bbb7823282e40847ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732105001807 2024-11-20T12:16:43,940 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting cea4f89403cb4bafaf13189fb657a3ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732105003023 2024-11-20T12:16:43,947 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#439 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:43,947 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/69230b26cc9644fab1e23e506ef673b4 is 50, key is test_row_0/C:col10/1732105003041/Put/seqid=0 2024-11-20T12:16:43,953 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/8252cc0b60cb47228d7c4bc66f78ff32 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/8252cc0b60cb47228d7c4bc66f78ff32 2024-11-20T12:16:43,957 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into 8252cc0b60cb47228d7c4bc66f78ff32(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:43,957 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:43,957 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=13, startTime=1732105003492; duration=0sec 2024-11-20T12:16:43,957 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:43,957 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742342_1518 (size=12949) 2024-11-20T12:16:43,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105063967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:43,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105063967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:43,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:44,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/167ff699b7d04e9b81c18c37ab6ef4f5 2024-11-20T12:16:44,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e235178b25cb4867be2dff019b56bc54 is 50, key is test_row_0/B:col10/1732105003687/Put/seqid=0 2024-11-20T12:16:44,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742343_1519 (size=12301) 2024-11-20T12:16:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-20T12:16:44,167 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:44,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-20T12:16:44,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:44,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:44,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105064187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105064187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:44,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T12:16:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,363 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/69230b26cc9644fab1e23e506ef673b4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/69230b26cc9644fab1e23e506ef673b4 2024-11-20T12:16:44,366 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into 69230b26cc9644fab1e23e506ef673b4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:44,366 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:44,366 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=12, startTime=1732105003492; duration=0sec 2024-11-20T12:16:44,366 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:44,366 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:44,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T12:16:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:44,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e235178b25cb4867be2dff019b56bc54 2024-11-20T12:16:44,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105064507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105064507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:44,519 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:44,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/a40b90da8c054d3d8335558c2dfb3e72 is 50, key is test_row_0/C:col10/1732105003687/Put/seqid=0 2024-11-20T12:16:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742344_1520 (size=12301) 2024-11-20T12:16:44,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/a40b90da8c054d3d8335558c2dfb3e72 2024-11-20T12:16:44,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/167ff699b7d04e9b81c18c37ab6ef4f5 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5 2024-11-20T12:16:44,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5, entries=350, sequenceid=345, filesize=21.5 K 2024-11-20T12:16:44,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/e235178b25cb4867be2dff019b56bc54 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54 2024-11-20T12:16:44,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54, entries=150, sequenceid=345, filesize=12.0 K 2024-11-20T12:16:44,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/a40b90da8c054d3d8335558c2dfb3e72 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72 2024-11-20T12:16:44,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72, entries=150, sequenceid=345, filesize=12.0 K 2024-11-20T12:16:44,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bf1698cac1d4eef2a4574349a5374fe2 in 879ms, sequenceid=345, compaction requested=true 2024-11-20T12:16:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:44,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:44,567 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:44,568 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:44,568 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47281 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:44,568 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:44,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:44,568 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,569 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/6b8d2a520df0453e864d98163ef3be5a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=46.2 K 2024-11-20T12:16:44,569 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:44,569 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:44,569 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,569 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/8252cc0b60cb47228d7c4bc66f78ff32, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=36.6 K 2024-11-20T12:16:44,569 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8d2a520df0453e864d98163ef3be5a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732105001807 2024-11-20T12:16:44,569 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8252cc0b60cb47228d7c4bc66f78ff32, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1732105001807 2024-11-20T12:16:44,570 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcfeaaa0ea5d440b91327b0284c95e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732105003023 2024-11-20T12:16:44,570 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e35d724f1d724504b9da4f17c8994bdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732105003023 2024-11-20T12:16:44,570 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167ff699b7d04e9b81c18c37ab6ef4f5, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732105003677 2024-11-20T12:16:44,570 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e235178b25cb4867be2dff019b56bc54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732105003685 2024-11-20T12:16:44,586 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:44,587 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/a9858f842d5e4658bb8060b7a3683061 is 50, key is test_row_0/B:col10/1732105003687/Put/seqid=0 2024-11-20T12:16:44,603 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:44,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/1b7e556b103a4370a6b8da6678c67ee0 is 50, key is test_row_0/A:col10/1732105003687/Put/seqid=0 2024-11-20T12:16:44,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742345_1521 (size=13017) 2024-11-20T12:16:44,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-20T12:16:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:44,628 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T12:16:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:44,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:44,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:44,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742346_1522 (size=13017) 2024-11-20T12:16:44,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/90e69b515e1d492c8af9e082af721230 is 50, key is test_row_0/A:col10/1732105003855/Put/seqid=0 2024-11-20T12:16:44,648 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/1b7e556b103a4370a6b8da6678c67ee0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b7e556b103a4370a6b8da6678c67ee0 2024-11-20T12:16:44,655 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 1b7e556b103a4370a6b8da6678c67ee0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:44,655 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:44,655 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=13, startTime=1732105004567; duration=0sec 2024-11-20T12:16:44,656 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:44,656 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:44,656 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T12:16:44,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742347_1523 (size=12301) 2024-11-20T12:16:44,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T12:16:44,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T12:16:44,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. because compaction request was cancelled 2024-11-20T12:16:44,661 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:44,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:45,016 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/a9858f842d5e4658bb8060b7a3683061 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/a9858f842d5e4658bb8060b7a3683061 2024-11-20T12:16:45,020 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into a9858f842d5e4658bb8060b7a3683061(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:45,020 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:45,020 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=13, startTime=1732105004567; duration=0sec 2024-11-20T12:16:45,020 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:45,020 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:45,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:45,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105065046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105065052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,061 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/90e69b515e1d492c8af9e082af721230 2024-11-20T12:16:45,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ef9cef42ba9e4805bf1e0458a3696227 is 50, key is test_row_0/B:col10/1732105003855/Put/seqid=0 2024-11-20T12:16:45,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742348_1524 (size=12301) 2024-11-20T12:16:45,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105065167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105065167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105065387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105065387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ef9cef42ba9e4805bf1e0458a3696227 2024-11-20T12:16:45,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8772fd1b926d4650a754dc028afe9e31 is 50, key is test_row_0/C:col10/1732105003855/Put/seqid=0 2024-11-20T12:16:45,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742349_1525 (size=12301) 2024-11-20T12:16:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105065696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105065696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:45,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:45,888 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8772fd1b926d4650a754dc028afe9e31 2024-11-20T12:16:45,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/90e69b515e1d492c8af9e082af721230 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230 2024-11-20T12:16:45,894 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230, entries=150, sequenceid=384, filesize=12.0 K 2024-11-20T12:16:45,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ef9cef42ba9e4805bf1e0458a3696227 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227 2024-11-20T12:16:45,896 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227, entries=150, sequenceid=384, filesize=12.0 K 2024-11-20T12:16:45,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/8772fd1b926d4650a754dc028afe9e31 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31 2024-11-20T12:16:45,900 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31, entries=150, sequenceid=384, filesize=12.0 K 2024-11-20T12:16:45,900 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bf1698cac1d4eef2a4574349a5374fe2 in 1272ms, sequenceid=384, compaction requested=true 2024-11-20T12:16:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-20T12:16:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-20T12:16:45,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-20T12:16:45,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-20T12:16:45,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.7360 sec 2024-11-20T12:16:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:46,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:46,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:46,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/eebc3cd1407440f392bd17a946fb787a is 50, key is test_row_0/A:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:46,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742350_1526 (size=14741) 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105006224, max timestamp=1732105006289Num uniques 859; 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105006224, max timestamp=1732105006289Num uniques 859; 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105006224, max timestamp=1732105006289Num uniques 859; 2024-11-20T12:16:46,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:16:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T12:16:46,316 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:46,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-20T12:16:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:46,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:46,318 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:46,319 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:46,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105066344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105066345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:46,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105066456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105066466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/eebc3cd1407440f392bd17a946fb787a 2024-11-20T12:16:46,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 is 50, key is test_row_0/B:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:46,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:46,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742351_1527 (size=12301) 2024-11-20T12:16:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:46,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105066666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105066686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:46,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:46,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:46,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:46,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105066977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105067007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 2024-11-20T12:16:47,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/bc44f8a849f244969ad5efc535d3ff04 is 50, key is test_row_0/C:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:47,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742352_1528 (size=12301) 2024-11-20T12:16:47,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/bc44f8a849f244969ad5efc535d3ff04 2024-11-20T12:16:47,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/eebc3cd1407440f392bd17a946fb787a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a 2024-11-20T12:16:47,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a, entries=200, sequenceid=397, filesize=14.4 K 2024-11-20T12:16:47,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 2024-11-20T12:16:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:47,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T12:16:47,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/bc44f8a849f244969ad5efc535d3ff04 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04 2024-11-20T12:16:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04, entries=150, sequenceid=397, filesize=12.0 K 2024-11-20T12:16:47,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bf1698cac1d4eef2a4574349a5374fe2 in 1250ms, sequenceid=397, compaction requested=true 2024-11-20T12:16:47,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:47,462 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:47,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:47,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:47,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:47,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:47,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:47,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bf1698cac1d4eef2a4574349a5374fe2:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:47,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:47,463 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:47,463 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/A is initiating minor compaction (all files) 2024-11-20T12:16:47,463 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/A in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,463 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b7e556b103a4370a6b8da6678c67ee0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=39.1 K 2024-11-20T12:16:47,463 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/B is initiating minor compaction (all files) 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b7e556b103a4370a6b8da6678c67ee0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732105003685 2024-11-20T12:16:47,464 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/B in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,464 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/a9858f842d5e4658bb8060b7a3683061, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=36.7 K 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e69b515e1d492c8af9e082af721230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732105003832 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a9858f842d5e4658bb8060b7a3683061, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732105003685 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting eebc3cd1407440f392bd17a946fb787a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732105005040 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting ef9cef42ba9e4805bf1e0458a3696227, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732105003832 2024-11-20T12:16:47,464 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3c40a4696a4fc4af4e6f77d19c6eb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732105005045 2024-11-20T12:16:47,470 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:47,470 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/9c669605a3334545a039be1fa15e7243 is 50, key is test_row_0/A:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:47,477 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#B#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:47,478 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/baf2588033bf46a482f08deac9685287 is 50, key is test_row_0/B:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742353_1529 (size=13119) 2024-11-20T12:16:47,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742354_1530 (size=13119) 2024-11-20T12:16:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:47,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:47,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:47,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/d6be288a67894899ab1d9c049596f33a is 50, key is test_row_0/A:col10/1732105006317/Put/seqid=0 2024-11-20T12:16:47,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742355_1531 (size=14741) 2024-11-20T12:16:47,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105067555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105067561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105067666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105067676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:47,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105067877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,887 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/9c669605a3334545a039be1fa15e7243 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/9c669605a3334545a039be1fa15e7243 2024-11-20T12:16:47,890 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/baf2588033bf46a482f08deac9685287 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/baf2588033bf46a482f08deac9685287 2024-11-20T12:16:47,890 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/A of bf1698cac1d4eef2a4574349a5374fe2 into 9c669605a3334545a039be1fa15e7243(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:47,890 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:47,890 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/A, priority=13, startTime=1732105007462; duration=0sec 2024-11-20T12:16:47,890 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:47,891 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:A 2024-11-20T12:16:47,891 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:16:47,891 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:16:47,891 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): bf1698cac1d4eef2a4574349a5374fe2/C is initiating minor compaction (all files) 2024-11-20T12:16:47,891 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bf1698cac1d4eef2a4574349a5374fe2/C in TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:47,891 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/69230b26cc9644fab1e23e506ef673b4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp, totalSize=48.7 K 2024-11-20T12:16:47,892 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69230b26cc9644fab1e23e506ef673b4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732105003023 2024-11-20T12:16:47,892 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40b90da8c054d3d8335558c2dfb3e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732105003685 2024-11-20T12:16:47,892 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8772fd1b926d4650a754dc028afe9e31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732105003832 2024-11-20T12:16:47,893 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc44f8a849f244969ad5efc535d3ff04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732105005045 2024-11-20T12:16:47,893 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/B of bf1698cac1d4eef2a4574349a5374fe2 into baf2588033bf46a482f08deac9685287(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:47,893 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:47,893 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/B, priority=13, startTime=1732105007462; duration=0sec 2024-11-20T12:16:47,894 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:47,894 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:B 2024-11-20T12:16:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105067886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:47,899 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bf1698cac1d4eef2a4574349a5374fe2#C#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:47,900 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/7299812ad0e048daae2f860ab23f85b4 is 50, key is test_row_0/C:col10/1732105006211/Put/seqid=0 2024-11-20T12:16:47,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742356_1532 (size=13085) 2024-11-20T12:16:47,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/d6be288a67894899ab1d9c049596f33a 2024-11-20T12:16:47,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/1a9b8ce5d30c40f9b821999b2f289336 is 50, key is test_row_0/B:col10/1732105006317/Put/seqid=0 2024-11-20T12:16:47,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742357_1533 (size=12301) 2024-11-20T12:16:48,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:48,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105068186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:48,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105068206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:48,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,308 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/7299812ad0e048daae2f860ab23f85b4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7299812ad0e048daae2f860ab23f85b4 2024-11-20T12:16:48,312 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bf1698cac1d4eef2a4574349a5374fe2/C of bf1698cac1d4eef2a4574349a5374fe2 into 7299812ad0e048daae2f860ab23f85b4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:16:48,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:48,312 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., storeName=bf1698cac1d4eef2a4574349a5374fe2/C, priority=12, startTime=1732105007463; duration=0sec 2024-11-20T12:16:48,312 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:48,313 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bf1698cac1d4eef2a4574349a5374fe2:C 2024-11-20T12:16:48,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/1a9b8ce5d30c40f9b821999b2f289336 2024-11-20T12:16:48,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/28c88472361a48c0a3ffdaac48fbafa2 is 50, key is test_row_0/C:col10/1732105006317/Put/seqid=0 2024-11-20T12:16:48,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742358_1534 (size=12301) 2024-11-20T12:16:48,400 INFO [Thread-2057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2057. 2024-11-20T12:16:48,400 DEBUG [Thread-2057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:48,400 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,400 DEBUG [Thread-2057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,400 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:48,402 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-11-20T12:16:48,402 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:48,402 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,402 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:48,403 INFO [Thread-2059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2059. 2024-11-20T12:16:48,403 DEBUG [Thread-2059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:48,403 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,403 DEBUG [Thread-2059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:48,412 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-11-20T12:16:48,412 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:48,412 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,412 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,412 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:48,413 INFO [Thread-2061 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2061. 2024-11-20T12:16:48,413 DEBUG [Thread-2061 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:48,413 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,413 DEBUG [Thread-2061 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:48,413 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:48,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:48,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:48,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43456 deadline: 1732105068697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43430 deadline: 1732105068727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 , the old value is region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., hostname=843d83d51ba0,42825,1732104864746, seqNum=2 because the exception is null or not the one we care about 2024-11-20T12:16:48,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/28c88472361a48c0a3ffdaac48fbafa2 2024-11-20T12:16:48,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/d6be288a67894899ab1d9c049596f33a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/d6be288a67894899ab1d9c049596f33a 2024-11-20T12:16:48,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/d6be288a67894899ab1d9c049596f33a, entries=200, sequenceid=433, filesize=14.4 K 2024-11-20T12:16:48,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/1a9b8ce5d30c40f9b821999b2f289336 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/1a9b8ce5d30c40f9b821999b2f289336 2024-11-20T12:16:48,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/1a9b8ce5d30c40f9b821999b2f289336, entries=150, sequenceid=433, filesize=12.0 K 2024-11-20T12:16:48,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/28c88472361a48c0a3ffdaac48fbafa2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/28c88472361a48c0a3ffdaac48fbafa2 2024-11-20T12:16:48,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/28c88472361a48c0a3ffdaac48fbafa2, entries=150, sequenceid=433, filesize=12.0 K 2024-11-20T12:16:48,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for bf1698cac1d4eef2a4574349a5374fe2 in 1247ms, sequenceid=433, compaction requested=false 2024-11-20T12:16:48,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:48,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-20T12:16:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:48,773 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:48,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:48,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/1b50aaf2e39a4dc2b04b121cd331039f is 50, key is test_row_0/A:col10/1732105007554/Put/seqid=0 2024-11-20T12:16:48,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742359_1535 (size=9857) 2024-11-20T12:16:49,185 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/1b50aaf2e39a4dc2b04b121cd331039f 2024-11-20T12:16:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/abed0120aeeb45639c037d5f761e4189 is 50, key is test_row_0/B:col10/1732105007554/Put/seqid=0 2024-11-20T12:16:49,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742360_1536 (size=9857) 2024-11-20T12:16:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. as already flushing 2024-11-20T12:16:49,501 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-11-20T12:16:49,501 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:49,501 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,502 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:49,520 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-11-20T12:16:49,520 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:49,520 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,520 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:49,538 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-11-20T12:16:49,538 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:49,538 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,538 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/abed0120aeeb45639c037d5f761e4189 2024-11-20T12:16:49,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f90ed7c9853e4d9fa4f3167a81a7f14a is 50, key is test_row_0/C:col10/1732105007554/Put/seqid=0 2024-11-20T12:16:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742361_1537 (size=9857) 2024-11-20T12:16:49,709 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-11-20T12:16:49,709 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:49,709 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,710 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,710 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:49,741 INFO [Thread-2048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2048. 2024-11-20T12:16:49,741 DEBUG [Thread-2048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:16:49,741 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,741 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:49,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:50,010 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f90ed7c9853e4d9fa4f3167a81a7f14a 2024-11-20T12:16:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/1b50aaf2e39a4dc2b04b121cd331039f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b50aaf2e39a4dc2b04b121cd331039f 2024-11-20T12:16:50,022 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b50aaf2e39a4dc2b04b121cd331039f, entries=100, sequenceid=446, filesize=9.6 K 2024-11-20T12:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/abed0120aeeb45639c037d5f761e4189 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/abed0120aeeb45639c037d5f761e4189 2024-11-20T12:16:50,026 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/abed0120aeeb45639c037d5f761e4189, entries=100, sequenceid=446, filesize=9.6 K 2024-11-20T12:16:50,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/f90ed7c9853e4d9fa4f3167a81a7f14a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f90ed7c9853e4d9fa4f3167a81a7f14a 2024-11-20T12:16:50,029 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f90ed7c9853e4d9fa4f3167a81a7f14a, entries=100, sequenceid=446, filesize=9.6 K 2024-11-20T12:16:50,030 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for bf1698cac1d4eef2a4574349a5374fe2 in 1257ms, sequenceid=446, compaction requested=true 2024-11-20T12:16:50,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for bf1698cac1d4eef2a4574349a5374fe2: 2024-11-20T12:16:50,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:50,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-20T12:16:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-20T12:16:50,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T12:16:50,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7140 sec 2024-11-20T12:16:50,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 3.7170 sec 2024-11-20T12:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-20T12:16:50,477 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 142 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 986 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2957 rows 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2988 rows 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2987 rows 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-11-20T12:16:50,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2984 rows 2024-11-20T12:16:50,477 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:16:50,477 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:16:50,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:50,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:50,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:16:50,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:50,481 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105010481"}]},"ts":"1732105010481"} 2024-11-20T12:16:50,482 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:16:50,482 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:16:50,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:16:50,483 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, UNASSIGN}] 2024-11-20T12:16:50,484 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, UNASSIGN 2024-11-20T12:16:50,485 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=bf1698cac1d4eef2a4574349a5374fe2, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:50,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, UNASSIGN because future has completed 2024-11-20T12:16:50,486 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:16:50,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:50,639 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:50,639 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:16:50,639 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing bf1698cac1d4eef2a4574349a5374fe2, disabling compactions & flushes 2024-11-20T12:16:50,639 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:50,639 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:50,639 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. after waiting 0 ms 2024-11-20T12:16:50,639 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:50,639 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing bf1698cac1d4eef2a4574349a5374fe2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=A 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=B 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bf1698cac1d4eef2a4574349a5374fe2, store=C 2024-11-20T12:16:50,640 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:50,648 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/bc0e2c37839b46d8b6412972c21662a8 is 50, key is test_row_0/A:col10/1732105009707/Put/seqid=0 2024-11-20T12:16:50,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742362_1538 (size=12301) 2024-11-20T12:16:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:51,053 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/bc0e2c37839b46d8b6412972c21662a8 2024-11-20T12:16:51,064 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ca980bca495b4ea8b3db57ca823f7758 is 50, key is test_row_0/B:col10/1732105009707/Put/seqid=0 2024-11-20T12:16:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742363_1539 (size=12301) 2024-11-20T12:16:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:51,470 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ca980bca495b4ea8b3db57ca823f7758 2024-11-20T12:16:51,482 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/46dfc783b23b4a56968c7fa97e84f182 is 50, key is test_row_0/C:col10/1732105009707/Put/seqid=0 2024-11-20T12:16:51,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742364_1540 (size=12301) 2024-11-20T12:16:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:51,887 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/46dfc783b23b4a56968c7fa97e84f182 2024-11-20T12:16:51,894 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/A/bc0e2c37839b46d8b6412972c21662a8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/bc0e2c37839b46d8b6412972c21662a8 2024-11-20T12:16:51,901 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/bc0e2c37839b46d8b6412972c21662a8, entries=150, sequenceid=454, filesize=12.0 K 2024-11-20T12:16:51,902 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/B/ca980bca495b4ea8b3db57ca823f7758 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ca980bca495b4ea8b3db57ca823f7758 2024-11-20T12:16:51,907 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ca980bca495b4ea8b3db57ca823f7758, entries=150, sequenceid=454, filesize=12.0 K 2024-11-20T12:16:51,908 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/.tmp/C/46dfc783b23b4a56968c7fa97e84f182 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/46dfc783b23b4a56968c7fa97e84f182 2024-11-20T12:16:51,913 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/46dfc783b23b4a56968c7fa97e84f182, entries=150, sequenceid=454, filesize=12.0 K 2024-11-20T12:16:51,914 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bf1698cac1d4eef2a4574349a5374fe2 in 1275ms, sequenceid=454, compaction requested=true 2024-11-20T12:16:51,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/5478c9cf01344680872a1c1ebd0cd04e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/59179126537e4d3089f0af54c2bf6787, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/cd03afcbdcc4457ca902ab0f063169e1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4a91a5ae0f504f01b15d73b46918db59, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/6b8d2a520df0453e864d98163ef3be5a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b7e556b103a4370a6b8da6678c67ee0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a] to archive 2024-11-20T12:16:51,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:51,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7846e0ec4bed40189775e124a002f443 2024-11-20T12:16:51,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/f40d466f67074c14b79b0dc5b2a876b6 2024-11-20T12:16:51,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/efd659fcec0c4963a1620ec74d06b08e 2024-11-20T12:16:51,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b5c5afd14bd3475f8433b8207a614272 2024-11-20T12:16:51,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/5478c9cf01344680872a1c1ebd0cd04e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/5478c9cf01344680872a1c1ebd0cd04e 2024-11-20T12:16:51,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/7d11f0ecb6f5407dabc619d0d4a58556 2024-11-20T12:16:51,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e5c69c290bda45009e79bb904e17c1a9 2024-11-20T12:16:51,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/2e08f5305aa640a7b639b3a1204defe9 2024-11-20T12:16:51,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/59179126537e4d3089f0af54c2bf6787 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/59179126537e4d3089f0af54c2bf6787 2024-11-20T12:16:51,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3d057461b83e491d98c9b2f6bff910ea 2024-11-20T12:16:51,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/3b18fcd2ce7746f394e6d22e45550388 2024-11-20T12:16:51,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4bf70b195a044bef95d5d30657320f24 2024-11-20T12:16:51,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/cd03afcbdcc4457ca902ab0f063169e1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/cd03afcbdcc4457ca902ab0f063169e1 2024-11-20T12:16:51,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/aa4e6403f9344a3a9b2917732a0ae098 2024-11-20T12:16:51,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/e927fe29e45e4744bf2d96ff894f4f03 2024-11-20T12:16:51,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4a91a5ae0f504f01b15d73b46918db59 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/4a91a5ae0f504f01b15d73b46918db59 2024-11-20T12:16:51,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/b06bfa74f4144d719907d0f18a2f00e7 2024-11-20T12:16:51,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/ea1e61cec45c40af9d03a7c2d7cc03f6 2024-11-20T12:16:51,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/6b8d2a520df0453e864d98163ef3be5a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/6b8d2a520df0453e864d98163ef3be5a 2024-11-20T12:16:51,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/dcfeaaa0ea5d440b91327b0284c95e72 2024-11-20T12:16:51,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/167ff699b7d04e9b81c18c37ab6ef4f5 2024-11-20T12:16:51,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b7e556b103a4370a6b8da6678c67ee0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b7e556b103a4370a6b8da6678c67ee0 2024-11-20T12:16:51,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/90e69b515e1d492c8af9e082af721230 2024-11-20T12:16:51,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/eebc3cd1407440f392bd17a946fb787a 2024-11-20T12:16:51,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/adafd6006c6c461a94180b3e13ef223b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fd77d5b0f5df4f2b93b5118985156f49, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9520d63ef54545e0adb79d5248474054, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2ee7718a2cc24b96b184bbfaeea700a1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/8252cc0b60cb47228d7c4bc66f78ff32, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/a9858f842d5e4658bb8060b7a3683061, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2] to archive 2024-11-20T12:16:51,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:51,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/d180bf1e53af411f8b788e8305ebb32a 2024-11-20T12:16:51,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/cf1f9a0017ee445b978b2bf9e2252a9a 2024-11-20T12:16:51,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/4199f37cdc0341f6a2edb8f5ff42e923 2024-11-20T12:16:51,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/adafd6006c6c461a94180b3e13ef223b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/adafd6006c6c461a94180b3e13ef223b 2024-11-20T12:16:51,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/c9dac18523fb450eadf34c7e12d7f49a 2024-11-20T12:16:51,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9c1801b74cbf4237a52108a8bb20fa66 2024-11-20T12:16:51,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9acad4c7e1cc4434aecc6eadf4ae63d3 2024-11-20T12:16:51,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fd77d5b0f5df4f2b93b5118985156f49 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fd77d5b0f5df4f2b93b5118985156f49 2024-11-20T12:16:51,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3bcb5045a6f7412ab5c8c3f47c20d8e2 2024-11-20T12:16:51,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0f627c9c2e034ab4ba7c81b32fc3cc80 2024-11-20T12:16:51,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e4fabf38dfa34038af88b185214a2047 2024-11-20T12:16:51,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9520d63ef54545e0adb79d5248474054 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/9520d63ef54545e0adb79d5248474054 2024-11-20T12:16:51,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/3354a9cdfe604fcfb1535c18289cadc2 2024-11-20T12:16:51,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/0b9d5026213b4a109a5f2f7be736ca30 2024-11-20T12:16:51,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2ee7718a2cc24b96b184bbfaeea700a1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2ee7718a2cc24b96b184bbfaeea700a1 2024-11-20T12:16:51,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/799f829d0ede440ca0574a326d77f546 2024-11-20T12:16:51,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/424e0056cae14936961dccee48b89497 2024-11-20T12:16:51,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/8252cc0b60cb47228d7c4bc66f78ff32 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/8252cc0b60cb47228d7c4bc66f78ff32 2024-11-20T12:16:51,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/fe61a02b44404d4f98181cb6abcfa6c4 2024-11-20T12:16:51,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e35d724f1d724504b9da4f17c8994bdb 2024-11-20T12:16:51,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/a9858f842d5e4658bb8060b7a3683061 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/a9858f842d5e4658bb8060b7a3683061 2024-11-20T12:16:51,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/e235178b25cb4867be2dff019b56bc54 2024-11-20T12:16:51,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ef9cef42ba9e4805bf1e0458a3696227 2024-11-20T12:16:51,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/2e3c40a4696a4fc4af4e6f77d19c6eb2 2024-11-20T12:16:51,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/26eec63e8ee54d0e8b92a671e32318ad, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9afcf333c16a4923bab61fe20451aa00, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f005fcdd198348e9829858879c1aa8e6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f9a05bc85f0c410d8e5f5d237c25fb8b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/69230b26cc9644fab1e23e506ef673b4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04] to archive 2024-11-20T12:16:51,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:16:51,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/aebea02ece9f4535b71c1401092740f8 2024-11-20T12:16:51,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/0c68a8928cfe4252a40508fd79ab25bf 2024-11-20T12:16:51,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4498abb271304bfdacb7d7e5796da03c 2024-11-20T12:16:51,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/26eec63e8ee54d0e8b92a671e32318ad to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/26eec63e8ee54d0e8b92a671e32318ad 2024-11-20T12:16:51,980 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8de86bd753c3452793e78e7508499ecf 2024-11-20T12:16:51,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/d866f0b01d374bd28f0a1964f2b289f7 2024-11-20T12:16:51,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/86c36b9e519e4263b0ec1aaa93827c09 2024-11-20T12:16:51,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9afcf333c16a4923bab61fe20451aa00 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9afcf333c16a4923bab61fe20451aa00 2024-11-20T12:16:51,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7a47c400775f490f9ceb4fb1cc4fb4d4 2024-11-20T12:16:51,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/9746dcee73d74824bc4a111ab835dc66 2024-11-20T12:16:51,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/3d9e7b519074454480f8863c475a44b3 2024-11-20T12:16:51,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f005fcdd198348e9829858879c1aa8e6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f005fcdd198348e9829858879c1aa8e6 2024-11-20T12:16:51,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/ceca058a765d48e390ce4539c9874753 2024-11-20T12:16:51,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/53d6a9a2fde64d7381f2c53234d24229 2024-11-20T12:16:51,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f9a05bc85f0c410d8e5f5d237c25fb8b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f9a05bc85f0c410d8e5f5d237c25fb8b 2024-11-20T12:16:51,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f4d5991f406545ed9638ea20d8bc9f77 2024-11-20T12:16:51,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/4c61d63ec8664181af9ea6a563fcb9e7 2024-11-20T12:16:51,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f157dcbdcc0f41bbb7823282e40847ab 2024-11-20T12:16:51,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/69230b26cc9644fab1e23e506ef673b4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/69230b26cc9644fab1e23e506ef673b4 2024-11-20T12:16:51,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/cea4f89403cb4bafaf13189fb657a3ce 2024-11-20T12:16:51,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/a40b90da8c054d3d8335558c2dfb3e72 2024-11-20T12:16:51,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/8772fd1b926d4650a754dc028afe9e31 2024-11-20T12:16:52,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/bc44f8a849f244969ad5efc535d3ff04 2024-11-20T12:16:52,004 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/recovered.edits/457.seqid, newMaxSeqId=457, maxSeqId=1 2024-11-20T12:16:52,004 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. 2024-11-20T12:16:52,005 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for bf1698cac1d4eef2a4574349a5374fe2: Waiting for close lock at 1732105010639Running coprocessor pre-close hooks at 1732105010639Disabling compacts and flushes for region at 1732105010639Disabling writes for close at 1732105010639Obtaining lock to block concurrent updates at 1732105010639Preparing flush snapshotting stores in bf1698cac1d4eef2a4574349a5374fe2 at 1732105010639Finished memstore snapshotting TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732105010640 (+1 ms)Flushing stores of TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2. at 1732105010641 (+1 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/A: creating writer at 1732105010641Flushing bf1698cac1d4eef2a4574349a5374fe2/A: appending metadata at 1732105010648 (+7 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/A: closing flushed file at 1732105010648Flushing bf1698cac1d4eef2a4574349a5374fe2/B: creating writer at 1732105011060 (+412 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/B: appending metadata at 1732105011064 (+4 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/B: closing flushed file at 1732105011064Flushing bf1698cac1d4eef2a4574349a5374fe2/C: creating writer at 1732105011479 (+415 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/C: appending metadata at 1732105011481 (+2 ms)Flushing bf1698cac1d4eef2a4574349a5374fe2/C: closing flushed file at 1732105011482 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f4d0762: reopening flushed file at 1732105011893 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa9c2fd: reopening flushed file at 1732105011901 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31e3b173: reopening flushed file at 1732105011907 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bf1698cac1d4eef2a4574349a5374fe2 in 1275ms, sequenceid=454, compaction requested=true at 1732105011914 (+7 ms)Writing region close event to WAL at 1732105012002 (+88 ms)Running coprocessor post-close hooks at 1732105012004 (+2 ms)Closed at 1732105012004 2024-11-20T12:16:52,006 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:52,007 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=bf1698cac1d4eef2a4574349a5374fe2, regionState=CLOSED 2024-11-20T12:16:52,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:52,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-20T12:16:52,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure bf1698cac1d4eef2a4574349a5374fe2, server=843d83d51ba0,42825,1732104864746 in 1.5220 sec 2024-11-20T12:16:52,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T12:16:52,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bf1698cac1d4eef2a4574349a5374fe2, UNASSIGN in 1.5260 sec 2024-11-20T12:16:52,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-20T12:16:52,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5290 sec 2024-11-20T12:16:52,013 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105012013"}]},"ts":"1732105012013"} 2024-11-20T12:16:52,014 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:16:52,014 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:16:52,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5360 sec 2024-11-20T12:16:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T12:16:52,638 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:52,640 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:16:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T12:16:52,647 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,653 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:52,655 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/recovered.edits] 2024-11-20T12:16:52,658 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b50aaf2e39a4dc2b04b121cd331039f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/1b50aaf2e39a4dc2b04b121cd331039f 2024-11-20T12:16:52,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/9c669605a3334545a039be1fa15e7243 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/9c669605a3334545a039be1fa15e7243 2024-11-20T12:16:52,660 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/bc0e2c37839b46d8b6412972c21662a8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/bc0e2c37839b46d8b6412972c21662a8 2024-11-20T12:16:52,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/d6be288a67894899ab1d9c049596f33a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/A/d6be288a67894899ab1d9c049596f33a 2024-11-20T12:16:52,664 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/1a9b8ce5d30c40f9b821999b2f289336 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/1a9b8ce5d30c40f9b821999b2f289336 2024-11-20T12:16:52,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/abed0120aeeb45639c037d5f761e4189 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/abed0120aeeb45639c037d5f761e4189 2024-11-20T12:16:52,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/baf2588033bf46a482f08deac9685287 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/baf2588033bf46a482f08deac9685287 2024-11-20T12:16:52,667 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ca980bca495b4ea8b3db57ca823f7758 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/B/ca980bca495b4ea8b3db57ca823f7758 2024-11-20T12:16:52,669 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/28c88472361a48c0a3ffdaac48fbafa2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/28c88472361a48c0a3ffdaac48fbafa2 2024-11-20T12:16:52,670 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/46dfc783b23b4a56968c7fa97e84f182 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/46dfc783b23b4a56968c7fa97e84f182 2024-11-20T12:16:52,671 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7299812ad0e048daae2f860ab23f85b4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/7299812ad0e048daae2f860ab23f85b4 2024-11-20T12:16:52,672 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f90ed7c9853e4d9fa4f3167a81a7f14a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/C/f90ed7c9853e4d9fa4f3167a81a7f14a 2024-11-20T12:16:52,674 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/recovered.edits/457.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2/recovered.edits/457.seqid 2024-11-20T12:16:52,674 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/bf1698cac1d4eef2a4574349a5374fe2 2024-11-20T12:16:52,674 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:16:52,676 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,678 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:16:52,680 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:16:52,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,681 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:16:52,681 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732105012681"}]},"ts":"9223372036854775807"} 2024-11-20T12:16:52,682 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:16:52,682 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bf1698cac1d4eef2a4574349a5374fe2, NAME => 'TestAcidGuarantees,,1732104986195.bf1698cac1d4eef2a4574349a5374fe2.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:16:52,682 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:16:52,682 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732105012682"}]},"ts":"9223372036854775807"} 2024-11-20T12:16:52,683 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:16:52,684 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-20T12:16:52,705 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:16:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-20T12:16:52,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:16:52,769 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:52,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 478), ProcessCount=11 (was 11), AvailableMemoryMB=2555 (was 2581) 2024-11-20T12:16:52,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=426, ProcessCount=11, AvailableMemoryMB=2555 2024-11-20T12:16:52,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:16:52,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:16:52,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:52,786 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:16:52,786 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:52,786 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-20T12:16:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:52,787 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:16:52,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742365_1541 (size=1061) 2024-11-20T12:16:52,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:53,245 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91 2024-11-20T12:16:53,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742366_1542 (size=53) 2024-11-20T12:16:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ddc2f1efcc9e364aaba38798681135ba, disabling compactions & flushes 2024-11-20T12:16:53,654 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. after waiting 0 ms 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,654 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,654 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ddc2f1efcc9e364aaba38798681135ba: Waiting for close lock at 1732105013654Disabling compacts and flushes for region at 1732105013654Disabling writes for close at 1732105013654Writing region close event to WAL at 1732105013654Closed at 1732105013654 2024-11-20T12:16:53,657 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:16:53,657 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732105013657"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732105013657"}]},"ts":"1732105013657"} 2024-11-20T12:16:53,660 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T12:16:53,662 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:16:53,662 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105013662"}]},"ts":"1732105013662"} 2024-11-20T12:16:53,665 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T12:16:53,665 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, ASSIGN}] 2024-11-20T12:16:53,667 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, ASSIGN 2024-11-20T12:16:53,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, ASSIGN; state=OFFLINE, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=false 2024-11-20T12:16:53,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:53,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, ASSIGN because future has completed 2024-11-20T12:16:53,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:53,982 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:16:53,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:16:53,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,983 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,984 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,986 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:53,986 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName A 2024-11-20T12:16:53,986 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:53,987 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:53,987 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,988 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:53,988 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName B 2024-11-20T12:16:53,988 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:53,989 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:53,989 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,990 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:53,990 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName C 2024-11-20T12:16:53,990 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:53,991 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:53,991 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:53,991 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,991 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,993 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,993 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,993 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:16:53,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,996 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:16:53,997 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened ddc2f1efcc9e364aaba38798681135ba; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59612708, jitterRate=-0.11170142889022827}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:16:53,997 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:53,997 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for ddc2f1efcc9e364aaba38798681135ba: Running coprocessor pre-open hook at 1732105013983Writing region info on filesystem at 1732105013983Initializing all the Stores at 1732105013984 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105013984Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105013984Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105013984Cleaning up temporary data from old regions at 1732105013993 (+9 ms)Running coprocessor post-open hooks at 1732105013997 (+4 ms)Region opened successfully at 1732105013997 2024-11-20T12:16:53,998 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., pid=154, masterSystemTime=1732105013978 2024-11-20T12:16:54,000 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:54,000 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:54,000 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:54,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:54,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-20T12:16:54,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 in 180 msec 2024-11-20T12:16:54,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T12:16:54,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, ASSIGN in 339 msec 2024-11-20T12:16:54,007 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:16:54,007 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105014007"}]},"ts":"1732105014007"} 2024-11-20T12:16:54,008 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T12:16:54,009 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:16:54,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2240 sec 2024-11-20T12:16:54,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T12:16:54,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:54,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a6775c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:54,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:54,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:54,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c812665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:54,941 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6398d18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:54,941 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:54,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:54,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:54,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,40015,1732104863721 2024-11-20T12:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d9d9088 2024-11-20T12:16:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:16:54,945 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:16:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T12:16:54,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:16:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T12:16:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742367_1543 (size=1097) 2024-11-20T12:16:55,356 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T12:16:55,356 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T12:16:55,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:16:55,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE}] 2024-11-20T12:16:55,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE 2024-11-20T12:16:55,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE because future has completed 2024-11-20T12:16:55,363 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:16:55,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:55,515 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,515 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:16:55,515 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing ddc2f1efcc9e364aaba38798681135ba, disabling compactions & flushes 2024-11-20T12:16:55,515 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,515 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,515 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. after waiting 0 ms 2024-11-20T12:16:55,515 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,518 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:16:55,519 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,519 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for ddc2f1efcc9e364aaba38798681135ba: Waiting for close lock at 1732105015515Running coprocessor pre-close hooks at 1732105015515Disabling compacts and flushes for region at 1732105015515Disabling writes for close at 1732105015515Writing region close event to WAL at 1732105015516 (+1 ms)Running coprocessor post-close hooks at 1732105015519 (+3 ms)Closed at 1732105015519 2024-11-20T12:16:55,519 WARN [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: ddc2f1efcc9e364aaba38798681135ba to self. 2024-11-20T12:16:55,520 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=CLOSED 2024-11-20T12:16:55,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:55,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-20T12:16:55,523 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 in 159 msec 2024-11-20T12:16:55,524 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE; state=CLOSED, location=843d83d51ba0,42825,1732104864746; forceNewPlan=false, retain=true 2024-11-20T12:16:55,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=OPENING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE because future has completed 2024-11-20T12:16:55,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:16:55,834 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,834 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:16:55,835 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,835 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:16:55,835 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,835 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,837 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,838 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:55,839 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName A 2024-11-20T12:16:55,841 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:55,841 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:55,842 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,843 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:55,843 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName B 2024-11-20T12:16:55,843 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:55,844 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:55,844 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,845 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T12:16:55,845 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddc2f1efcc9e364aaba38798681135ba columnFamilyName C 2024-11-20T12:16:55,845 DEBUG [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:55,845 INFO [StoreOpener-ddc2f1efcc9e364aaba38798681135ba-1 {}] regionserver.HStore(327): Store=ddc2f1efcc9e364aaba38798681135ba/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:16:55,846 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,846 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,846 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,847 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,848 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,848 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,849 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T12:16:55,850 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,851 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened ddc2f1efcc9e364aaba38798681135ba; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72593296, jitterRate=0.08172440528869629}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T12:16:55,851 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,851 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for ddc2f1efcc9e364aaba38798681135ba: Running coprocessor pre-open hook at 1732105015835Writing region info on filesystem at 1732105015835Initializing all the Stores at 1732105015837 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105015837Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105015837Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732105015837Cleaning up temporary data from old regions at 1732105015848 (+11 ms)Running coprocessor post-open hooks at 1732105015851 (+3 ms)Region opened successfully at 1732105015851 2024-11-20T12:16:55,852 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., pid=159, masterSystemTime=1732105015828 2024-11-20T12:16:55,854 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,854 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:55,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=OPEN, openSeqNum=5, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,855 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:16:55,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-20T12:16:55,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 in 180 msec 2024-11-20T12:16:55,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T12:16:55,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, REOPEN/MOVE in 498 msec 2024-11-20T12:16:55,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-20T12:16:55,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-20T12:16:55,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-20T12:16:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-20T12:16:55,868 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:16:55,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d987c42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,870 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772c8cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,872 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6894d628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af38ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,874 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e06b79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,876 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c695c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eb37c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a13556, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@474c9aa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,882 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d415d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,882 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,882 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fba629c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,884 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,884 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d63e3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e839bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,886 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21168f63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,886 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d85d6a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbb80ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10dd51f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f1df9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5df76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,894 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0fb215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,895 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,895 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,896 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af03a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d8e7d65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,898 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d580189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,900 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e9bcf4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,40015,-1 for getting cluster id 2024-11-20T12:16:55,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:16:55,902 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8050fc35-0ca3-4aaf-8d5d-0952e660f100' 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8050fc35-0ca3-4aaf-8d5d-0952e660f100" 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fa003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,40015,-1] 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:16:55,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:16:55,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:16:55,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1dcb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:16:55,904 DEBUG [Thread-2414 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,904 DEBUG [Thread-2425 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,904 DEBUG [Thread-2416 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,904 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,904 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,905 DEBUG [Thread-2418 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,905 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,906 DEBUG [Thread-2420 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,905 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T12:16:55,906 DEBUG [Thread-2422 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,907 DEBUG [Thread-2422 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,907 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:16:55,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,907 DEBUG [Thread-2422 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,907 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:16:55,907 DEBUG [Thread-2427 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,908 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:16:55,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,909 DEBUG [Thread-2427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,42825,1732104864746, seqNum=-1] 2024-11-20T12:16:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:16:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,910 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,913 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:16:55,913 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5] 2024-11-20T12:16:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:16:55,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:55,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105075932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105075934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105075935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732105015925, max timestamp=1732105015934Num uniques 859; 2024-11-20T12:16:55,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732105015925, max timestamp=1732105015934Num uniques 859; 2024-11-20T12:16:55,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732105015925, max timestamp=1732105015934Num uniques 859; 2024-11-20T12:16:55,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:16:55,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:16:55,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T12:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:55,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:55,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120830f9dde35724dddac3e4e11914640ea_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105015914/Put/seqid=0 2024-11-20T12:16:55,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742368_1544 (size=12154) 2024-11-20T12:16:55,951 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:55,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120830f9dde35724dddac3e4e11914640ea_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120830f9dde35724dddac3e4e11914640ea_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:55,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c82540f404c3454b9b141a9141c6e314, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:55,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c82540f404c3454b9b141a9141c6e314 is 175, key is test_row_0/A:col10/1732105015914/Put/seqid=0 2024-11-20T12:16:55,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742369_1545 (size=30955) 2024-11-20T12:16:55,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105075971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:55,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105075972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:56,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105076046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105076047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105076047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105076086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105076087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105076257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105076257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105076257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105076296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105076296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,360 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c82540f404c3454b9b141a9141c6e314 2024-11-20T12:16:56,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/3fdd30c484e8450f959a59221097b998 is 50, key is test_row_0/B:col10/1732105015914/Put/seqid=0 2024-11-20T12:16:56,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742370_1546 (size=12001) 2024-11-20T12:16:56,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:56,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105076566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105076566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105076567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105076607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:56,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105076607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:56,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/3fdd30c484e8450f959a59221097b998 2024-11-20T12:16:56,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/14d2c9d631be433da95931ab48846490 is 50, key is test_row_0/C:col10/1732105015914/Put/seqid=0 2024-11-20T12:16:56,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742371_1547 (size=12001) 2024-11-20T12:16:56,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:56,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:56,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:57,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105077077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:57,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105077077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:57,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105077077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105077116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105077116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:57,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:57,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:57,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:16:57,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/14d2c9d631be433da95931ab48846490 2024-11-20T12:16:57,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c82540f404c3454b9b141a9141c6e314 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314 2024-11-20T12:16:57,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314, entries=150, sequenceid=15, filesize=30.2 K 2024-11-20T12:16:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/3fdd30c484e8450f959a59221097b998 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998 2024-11-20T12:16:57,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T12:16:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/14d2c9d631be433da95931ab48846490 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490 2024-11-20T12:16:57,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T12:16:57,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ddc2f1efcc9e364aaba38798681135ba in 1299ms, sequenceid=15, compaction requested=false 2024-11-20T12:16:57,216 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T12:16:57,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:16:57,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T12:16:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:57,285 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:16:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:16:57,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112088f65b52cd4d4fb8a9d83aa964b9c3f4_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105015963/Put/seqid=0 2024-11-20T12:16:57,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742372_1548 (size=12154) 2024-11-20T12:16:57,524 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:16:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:57,697 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112088f65b52cd4d4fb8a9d83aa964b9c3f4_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088f65b52cd4d4fb8a9d83aa964b9c3f4_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/938e66eef9a84e8bb7110bb10c08e671, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:57,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/938e66eef9a84e8bb7110bb10c08e671 is 175, key is test_row_0/A:col10/1732105015963/Put/seqid=0 2024-11-20T12:16:57,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742373_1549 (size=30955) 2024-11-20T12:16:58,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:16:58,102 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/938e66eef9a84e8bb7110bb10c08e671 2024-11-20T12:16:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c9b38442056a48318bac37f5efeb63a0 is 50, key is test_row_0/B:col10/1732105015963/Put/seqid=0 2024-11-20T12:16:58,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742374_1550 (size=12001) 2024-11-20T12:16:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:58,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:16:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105078132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105078133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105078133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,137 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105078134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105078137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,141 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:16:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105078247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105078247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105078247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105078247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105078457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105078457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105078457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105078457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,511 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c9b38442056a48318bac37f5efeb63a0 2024-11-20T12:16:58,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/1f61cd4d7ccd472ca44db637225ba59b is 50, key is test_row_0/C:col10/1732105015963/Put/seqid=0 2024-11-20T12:16:58,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742375_1551 (size=12001) 2024-11-20T12:16:58,522 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/1f61cd4d7ccd472ca44db637225ba59b 2024-11-20T12:16:58,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/938e66eef9a84e8bb7110bb10c08e671 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671 2024-11-20T12:16:58,530 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671, entries=150, sequenceid=49, filesize=30.2 K 2024-11-20T12:16:58,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c9b38442056a48318bac37f5efeb63a0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0 2024-11-20T12:16:58,533 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:16:58,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/1f61cd4d7ccd472ca44db637225ba59b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b 2024-11-20T12:16:58,536 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-20T12:16:58,537 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=67.09 KB/68700 for ddc2f1efcc9e364aaba38798681135ba in 1252ms, sequenceid=49, compaction requested=false 2024-11-20T12:16:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:16:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:58,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T12:16:58,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T12:16:58,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T12:16:58,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-20T12:16:58,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.6350 sec 2024-11-20T12:16:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:58,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:16:58,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:58,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fadfeff03dac44b2b02d3e2c0a6f2a4d_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:58,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742376_1552 (size=12154) 2024-11-20T12:16:58,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105078786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105078789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105078790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105078791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105078896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105078897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105078897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105078907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:58,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105079107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105079107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105079107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105079116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:59,189 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fadfeff03dac44b2b02d3e2c0a6f2a4d_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fadfeff03dac44b2b02d3e2c0a6f2a4d_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:59,189 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/b70df3d31c6949db90ce5fed743039e0, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/b70df3d31c6949db90ce5fed743039e0 is 175, key is test_row_0/A:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:59,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742377_1553 (size=30955) 2024-11-20T12:16:59,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/b70df3d31c6949db90ce5fed743039e0 2024-11-20T12:16:59,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/86c780967e284ae2a91277658bea8ad1 is 50, key is test_row_0/B:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:59,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742378_1554 (size=12001) 2024-11-20T12:16:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105079417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105079417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105079417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105079426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/86c780967e284ae2a91277658bea8ad1 2024-11-20T12:16:59,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd9d93a52f154e35b28d03ad82e8a26b is 50, key is test_row_0/C:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742379_1555 (size=12001) 2024-11-20T12:16:59,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd9d93a52f154e35b28d03ad82e8a26b 2024-11-20T12:16:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/b70df3d31c6949db90ce5fed743039e0 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0 2024-11-20T12:16:59,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0, entries=150, sequenceid=63, filesize=30.2 K 2024-11-20T12:16:59,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/86c780967e284ae2a91277658bea8ad1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1 2024-11-20T12:16:59,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1, entries=150, sequenceid=63, filesize=11.7 K 2024-11-20T12:16:59,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd9d93a52f154e35b28d03ad82e8a26b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b 2024-11-20T12:16:59,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b, entries=150, sequenceid=63, filesize=11.7 K 2024-11-20T12:16:59,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ddc2f1efcc9e364aaba38798681135ba in 949ms, sequenceid=63, compaction requested=true 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:16:59,719 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:16:59,719 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:16:59,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:16:59,720 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:59,720 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:16:59,720 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:59,720 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=90.7 K 2024-11-20T12:16:59,720 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:59,720 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0] 2024-11-20T12:16:59,721 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:16:59,721 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:16:59,721 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:16:59,721 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=35.2 K 2024-11-20T12:16:59,722 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82540f404c3454b9b141a9141c6e314, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732105015914 2024-11-20T12:16:59,722 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fdd30c484e8450f959a59221097b998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732105015914 2024-11-20T12:16:59,722 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b38442056a48318bac37f5efeb63a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732105015961 2024-11-20T12:16:59,722 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938e66eef9a84e8bb7110bb10c08e671, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732105015961 2024-11-20T12:16:59,722 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c780967e284ae2a91277658bea8ad1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:16:59,724 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70df3d31c6949db90ce5fed743039e0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:16:59,743 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,745 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:59,745 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/39d21ec7c612414d924b4ad74ecf337a is 50, key is test_row_0/B:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:59,748 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411206f2873750bb2437f87ce6d2efcebbf89_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,750 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411206f2873750bb2437f87ce6d2efcebbf89_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,750 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206f2873750bb2437f87ce6d2efcebbf89_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742380_1556 (size=12104) 2024-11-20T12:16:59,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742381_1557 (size=4469) 2024-11-20T12:16:59,787 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#472 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:16:59,788 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/8c143199e4f94bb2bea9bdefe9bf17d2 is 175, key is test_row_0/A:col10/1732105018132/Put/seqid=0 2024-11-20T12:16:59,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742382_1558 (size=31058) 2024-11-20T12:16:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:59,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:16:59,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:16:59,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ab1a3f65e29549cd8e8d56dce1be70d0_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105018790/Put/seqid=0 2024-11-20T12:16:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742383_1559 (size=14594) 2024-11-20T12:16:59,941 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:16:59,952 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ab1a3f65e29549cd8e8d56dce1be70d0_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ab1a3f65e29549cd8e8d56dce1be70d0_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:16:59,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/5ee78531322e4937935880132a259c80, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:16:59,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/5ee78531322e4937935880132a259c80 is 175, key is test_row_0/A:col10/1732105018790/Put/seqid=0 2024-11-20T12:16:59,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105079951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105079951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105079953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:16:59,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105079954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:16:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:16:59,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:16:59,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742384_1560 (size=39549) 2024-11-20T12:16:59,977 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/5ee78531322e4937935880132a259c80 2024-11-20T12:16:59,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/b3c41193e0b449a2b88a321307c38b6f is 50, key is test_row_0/B:col10/1732105018790/Put/seqid=0 2024-11-20T12:17:00,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742385_1561 (size=12001) 2024-11-20T12:17:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T12:17:00,067 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:00,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T12:17:00,070 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:00,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:00,072 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:00,072 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105080067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105080067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105080067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105080067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,074 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105080147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,151 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:00,181 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/39d21ec7c612414d924b4ad74ecf337a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/39d21ec7c612414d924b4ad74ecf337a 2024-11-20T12:17:00,187 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into 39d21ec7c612414d924b4ad74ecf337a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:00,187 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:00,188 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=13, startTime=1732105019719; duration=0sec 2024-11-20T12:17:00,188 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:00,188 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:00,188 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:00,189 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:00,189 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:00,189 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,189 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=35.2 K 2024-11-20T12:17:00,189 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d2c9d631be433da95931ab48846490, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732105015914 2024-11-20T12:17:00,191 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f61cd4d7ccd472ca44db637225ba59b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732105015961 2024-11-20T12:17:00,191 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9d93a52f154e35b28d03ad82e8a26b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:17:00,201 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/8c143199e4f94bb2bea9bdefe9bf17d2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2 2024-11-20T12:17:00,208 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into 8c143199e4f94bb2bea9bdefe9bf17d2(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:00,208 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:00,208 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=13, startTime=1732105019719; duration=0sec 2024-11-20T12:17:00,208 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:00,208 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:00,217 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:00,217 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/28423eb99f9240fe9a25ca23668744f8 is 50, key is test_row_0/C:col10/1732105018132/Put/seqid=0 2024-11-20T12:17:00,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T12:17:00,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742386_1562 (size=12104) 2024-11-20T12:17:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105080278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105080278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105080278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105080278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T12:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:00,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/b3c41193e0b449a2b88a321307c38b6f 2024-11-20T12:17:00,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/cd143600671c4fb4b0f22bae4b2c5686 is 50, key is test_row_0/C:col10/1732105018790/Put/seqid=0 2024-11-20T12:17:00,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742387_1563 (size=12001) 2024-11-20T12:17:00,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/cd143600671c4fb4b0f22bae4b2c5686 2024-11-20T12:17:00,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/5ee78531322e4937935880132a259c80 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80 2024-11-20T12:17:00,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80, entries=200, sequenceid=82, filesize=38.6 K 2024-11-20T12:17:00,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/b3c41193e0b449a2b88a321307c38b6f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f 2024-11-20T12:17:00,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T12:17:00,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/cd143600671c4fb4b0f22bae4b2c5686 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686 2024-11-20T12:17:00,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686, entries=150, sequenceid=82, filesize=11.7 K 2024-11-20T12:17:00,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ddc2f1efcc9e364aaba38798681135ba in 581ms, sequenceid=82, compaction requested=false 2024-11-20T12:17:00,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:00,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:00,534 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:00,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:00,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e8e0410ac924f1d873395341d074b4a_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105019952/Put/seqid=0 2024-11-20T12:17:00,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742388_1564 (size=12154) 2024-11-20T12:17:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:00,615 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209e8e0410ac924f1d873395341d074b4a_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e8e0410ac924f1d873395341d074b4a_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/ee6e91fa17ce4e49a792f59ad46c38b7, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/ee6e91fa17ce4e49a792f59ad46c38b7 is 175, key is test_row_0/A:col10/1732105019952/Put/seqid=0 2024-11-20T12:17:00,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105080616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105080621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105080625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105080625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742389_1565 (size=30955) 2024-11-20T12:17:00,642 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/ee6e91fa17ce4e49a792f59ad46c38b7 2024-11-20T12:17:00,675 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/28423eb99f9240fe9a25ca23668744f8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/28423eb99f9240fe9a25ca23668744f8 2024-11-20T12:17:00,684 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into 28423eb99f9240fe9a25ca23668744f8(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:00,685 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:00,685 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=13, startTime=1732105019719; duration=0sec 2024-11-20T12:17:00,685 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:00,685 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:00,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/57d4f96c53854107b0c2dbbc25a7ba91 is 50, key is test_row_0/B:col10/1732105019952/Put/seqid=0 2024-11-20T12:17:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:00,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742390_1566 (size=12001) 2024-11-20T12:17:00,715 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/57d4f96c53854107b0c2dbbc25a7ba91 2024-11-20T12:17:00,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e77ca1bc68ec4d1b8128197cebefca63 is 50, key is test_row_0/C:col10/1732105019952/Put/seqid=0 2024-11-20T12:17:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105080727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,732 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105080737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105080737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105080737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742391_1567 (size=12001) 2024-11-20T12:17:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105080938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105080947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105080947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,950 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:00,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:00,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105080947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:00,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,183 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e77ca1bc68ec4d1b8128197cebefca63 2024-11-20T12:17:01,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/ee6e91fa17ce4e49a792f59ad46c38b7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7 2024-11-20T12:17:01,194 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7, entries=150, sequenceid=98, filesize=30.2 K 2024-11-20T12:17:01,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/57d4f96c53854107b0c2dbbc25a7ba91 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91 2024-11-20T12:17:01,200 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T12:17:01,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e77ca1bc68ec4d1b8128197cebefca63 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63 2024-11-20T12:17:01,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:01,210 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63, entries=150, sequenceid=98, filesize=11.7 K 2024-11-20T12:17:01,211 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ddc2f1efcc9e364aaba38798681135ba in 677ms, sequenceid=98, compaction requested=true 2024-11-20T12:17:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T12:17:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T12:17:01,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T12:17:01,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1410 sec 2024-11-20T12:17:01,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.1470 sec 2024-11-20T12:17:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:01,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:17:01,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:01,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202a262cbef00f4393a53e217e60d412a6_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:01,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742392_1568 (size=14594) 2024-11-20T12:17:01,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105081267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105081301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105081301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105081301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105081407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105081417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105081417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105081417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105081617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105081626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105081626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105081626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,678 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:01,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202a262cbef00f4393a53e217e60d412a6_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a262cbef00f4393a53e217e60d412a6_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:01,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/2c42592958f74cef8daace275e62fe61, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:01,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/2c42592958f74cef8daace275e62fe61 is 175, key is test_row_0/A:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:01,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742393_1569 (size=39549) 2024-11-20T12:17:01,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105081926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105081937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105081937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:01,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105081937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,085 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/2c42592958f74cef8daace275e62fe61 2024-11-20T12:17:02,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c8ff8d906ec1413b9d4385877711255a is 50, key is test_row_0/B:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:02,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742394_1570 (size=12001) 2024-11-20T12:17:02,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c8ff8d906ec1413b9d4385877711255a 2024-11-20T12:17:02,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd243e84c879415abd71516f9e97e46a is 50, key is test_row_0/C:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:02,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742395_1571 (size=12001) 2024-11-20T12:17:02,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd243e84c879415abd71516f9e97e46a 2024-11-20T12:17:02,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/2c42592958f74cef8daace275e62fe61 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61 2024-11-20T12:17:02,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61, entries=200, sequenceid=119, filesize=38.6 K 2024-11-20T12:17:02,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c8ff8d906ec1413b9d4385877711255a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a 2024-11-20T12:17:02,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T12:17:02,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/fd243e84c879415abd71516f9e97e46a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a 2024-11-20T12:17:02,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T12:17:02,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ddc2f1efcc9e364aaba38798681135ba in 904ms, sequenceid=119, compaction requested=true 2024-11-20T12:17:02,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:02,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:17:02,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:02,154 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:02,154 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:02,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:17:02,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:02,155 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:02,155 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:17:02,156 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:02,156 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/39d21ec7c612414d924b4ad74ecf337a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=47.0 K 2024-11-20T12:17:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:17:02,156 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d21ec7c612414d924b4ad74ecf337a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:17:02,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:02,156 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:02,156 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:17:02,156 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:02,156 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=137.8 K 2024-11-20T12:17:02,156 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:02,156 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61] 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c143199e4f94bb2bea9bdefe9bf17d2, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b3c41193e0b449a2b88a321307c38b6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732105018788 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee78531322e4937935880132a259c80, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732105018782 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d4f96c53854107b0c2dbbc25a7ba91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732105019949 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting c8ff8d906ec1413b9d4385877711255a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:02,157 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6e91fa17ce4e49a792f59ad46c38b7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732105019949 2024-11-20T12:17:02,158 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c42592958f74cef8daace275e62fe61, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:02,166 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:02,166 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/d940e9a1877b4a188437f9dd908b225a is 50, key is test_row_0/B:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:02,169 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:02,185 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202d6c2cc3de924cb285cc0f569a6330a6_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:02,188 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202d6c2cc3de924cb285cc0f569a6330a6_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:02,188 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202d6c2cc3de924cb285cc0f569a6330a6_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:02,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742396_1572 (size=12241) 2024-11-20T12:17:02,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742397_1573 (size=4469) 2024-11-20T12:17:02,206 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#484 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:02,206 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/67a81a6853604c9aa6e578d20aa7522d is 175, key is test_row_0/A:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:02,209 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/d940e9a1877b4a188437f9dd908b225a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/d940e9a1877b4a188437f9dd908b225a 2024-11-20T12:17:02,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742398_1574 (size=31195) 2024-11-20T12:17:02,215 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into d940e9a1877b4a188437f9dd908b225a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:02,215 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:02,215 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=12, startTime=1732105022154; duration=0sec 2024-11-20T12:17:02,215 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:02,215 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:02,215 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:02,216 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/67a81a6853604c9aa6e578d20aa7522d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d 2024-11-20T12:17:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T12:17:02,217 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:02,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-20T12:17:02,218 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:02,218 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:02,218 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:02,219 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/28423eb99f9240fe9a25ca23668744f8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=47.0 K 2024-11-20T12:17:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T12:17:02,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:02,220 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 28423eb99f9240fe9a25ca23668744f8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732105018132 2024-11-20T12:17:02,221 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cd143600671c4fb4b0f22bae4b2c5686, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1732105018788 2024-11-20T12:17:02,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:02,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:02,221 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e77ca1bc68ec4d1b8128197cebefca63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732105019949 2024-11-20T12:17:02,221 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting fd243e84c879415abd71516f9e97e46a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:02,226 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into 67a81a6853604c9aa6e578d20aa7522d(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:02,226 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:02,226 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=12, startTime=1732105022154; duration=0sec 2024-11-20T12:17:02,227 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:02,227 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:02,235 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#485 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:02,236 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/5e120c6053e649a9a023b6ad2614ab1c is 50, key is test_row_0/C:col10/1732105020622/Put/seqid=0 2024-11-20T12:17:02,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742399_1575 (size=12241) 2024-11-20T12:17:02,271 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/5e120c6053e649a9a023b6ad2614ab1c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e120c6053e649a9a023b6ad2614ab1c 2024-11-20T12:17:02,277 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into 5e120c6053e649a9a023b6ad2614ab1c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:02,277 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:02,277 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=12, startTime=1732105022154; duration=0sec 2024-11-20T12:17:02,277 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:02,277 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T12:17:02,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:02,374 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:02,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:02,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ad9f5ea8af44e3f8cddb4fc47a417bb_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105021270/Put/seqid=0 2024-11-20T12:17:02,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742400_1576 (size=12254) 2024-11-20T12:17:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:02,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:02,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105082469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105022461, max timestamp=1732105022477Num uniques 859; 2024-11-20T12:17:02,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:17:02,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105022461, max timestamp=1732105022477Num uniques 859; 2024-11-20T12:17:02,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:17:02,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732105022461, max timestamp=1732105022477Num uniques 859; 2024-11-20T12:17:02,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/843d83d51ba0:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T12:17:02,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105082523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105082525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105082530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T12:17:02,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105082596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105082637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105082637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105082646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:02,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105082806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,812 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ad9f5ea8af44e3f8cddb4fc47a417bb_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ad9f5ea8af44e3f8cddb4fc47a417bb_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/99ba1fc773b2444ab0befdebb378820d, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:02,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/99ba1fc773b2444ab0befdebb378820d is 175, key is test_row_0/A:col10/1732105021270/Put/seqid=0 2024-11-20T12:17:02,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742401_1577 (size=31055) 2024-11-20T12:17:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T12:17:02,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105082847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105082847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:02,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:02,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105082857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:02,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105083116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105083157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105083157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105083167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,216 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/99ba1fc773b2444ab0befdebb378820d 2024-11-20T12:17:03,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/1b80da6335ad42de9c567759929f72f1 is 50, key is test_row_0/B:col10/1732105021270/Put/seqid=0 2024-11-20T12:17:03,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742402_1578 (size=12101) 2024-11-20T12:17:03,226 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/1b80da6335ad42de9c567759929f72f1 2024-11-20T12:17:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/220d1e5eefc548c29c19fdc9722d675e is 50, key is test_row_0/C:col10/1732105021270/Put/seqid=0 2024-11-20T12:17:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742403_1579 (size=12101) 2024-11-20T12:17:03,251 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/220d1e5eefc548c29c19fdc9722d675e 2024-11-20T12:17:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/99ba1fc773b2444ab0befdebb378820d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d 2024-11-20T12:17:03,260 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d, entries=150, sequenceid=134, filesize=30.3 K 2024-11-20T12:17:03,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/1b80da6335ad42de9c567759929f72f1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1 2024-11-20T12:17:03,264 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T12:17:03,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/220d1e5eefc548c29c19fdc9722d675e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e 2024-11-20T12:17:03,268 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e, entries=150, sequenceid=134, filesize=11.8 K 2024-11-20T12:17:03,268 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ddc2f1efcc9e364aaba38798681135ba in 894ms, sequenceid=134, compaction requested=false 2024-11-20T12:17:03,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:03,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-20T12:17:03,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-20T12:17:03,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T12:17:03,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0490 sec 2024-11-20T12:17:03,273 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.0550 sec 2024-11-20T12:17:03,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T12:17:03,357 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:03,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-20T12:17:03,359 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:03,360 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:03,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:03,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:03,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-20T12:17:03,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:03,512 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:03,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:03,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112065bcc5c8ca224720b73eb2cfbae3529f_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105022522/Put/seqid=0 2024-11-20T12:17:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742404_1580 (size=12304) 2024-11-20T12:17:03,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:03,535 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112065bcc5c8ca224720b73eb2cfbae3529f_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065bcc5c8ca224720b73eb2cfbae3529f_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/82aef01f653b422098aa61364a956fbb, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/82aef01f653b422098aa61364a956fbb is 175, key is test_row_0/A:col10/1732105022522/Put/seqid=0 2024-11-20T12:17:03,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742405_1581 (size=31105) 2024-11-20T12:17:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:03,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105083675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105083676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105083676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105083676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:03,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105083786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:03,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:03,943 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/82aef01f653b422098aa61364a956fbb 2024-11-20T12:17:03,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6bca0abfea9340319ffff2e59f402fd6 is 50, key is test_row_0/B:col10/1732105022522/Put/seqid=0 2024-11-20T12:17:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742406_1582 (size=12151) 2024-11-20T12:17:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:04,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105083996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105084177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,180 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105084307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,310 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,375 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6bca0abfea9340319ffff2e59f402fd6 2024-11-20T12:17:04,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8c9e4e5688684db5b6899a1d8dd9ee2d is 50, key is test_row_0/C:col10/1732105022522/Put/seqid=0 2024-11-20T12:17:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742407_1583 (size=12151) 2024-11-20T12:17:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105084687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105084687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,687 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,688 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_0' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105084697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,700 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:04,783 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8c9e4e5688684db5b6899a1d8dd9ee2d 2024-11-20T12:17:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/82aef01f653b422098aa61364a956fbb as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb 2024-11-20T12:17:04,791 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb, entries=150, sequenceid=168, filesize=30.4 K 2024-11-20T12:17:04,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6bca0abfea9340319ffff2e59f402fd6 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6 2024-11-20T12:17:04,795 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6, entries=150, sequenceid=168, filesize=11.9 K 2024-11-20T12:17:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8c9e4e5688684db5b6899a1d8dd9ee2d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d 2024-11-20T12:17:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d, entries=150, sequenceid=168, filesize=11.9 K 2024-11-20T12:17:04,801 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for ddc2f1efcc9e364aaba38798681135ba in 1290ms, sequenceid=168, compaction requested=true 2024-11-20T12:17:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:04,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-20T12:17:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-20T12:17:04,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T12:17:04,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4440 sec 2024-11-20T12:17:04,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.4490 sec 2024-11-20T12:17:04,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:04,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:04,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:04,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d0e1579183dd4329bd33c3407d610a31_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:04,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742408_1584 (size=14794) 2024-11-20T12:17:04,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105084900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:05,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105085016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105085226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:05,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:05,246 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d0e1579183dd4329bd33c3407d610a31_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0e1579183dd4329bd33c3407d610a31_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:05,246 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/bed24805c37749c995f612b44d542bf2, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:05,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/bed24805c37749c995f612b44d542bf2 is 175, key is test_row_0/A:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742409_1585 (size=39749) 2024-11-20T12:17:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T12:17:05,507 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:05,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-20T12:17:05,510 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:05,510 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:05,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:05,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105085536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:05,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/bed24805c37749c995f612b44d542bf2 2024-11-20T12:17:05,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/670f82c4dc014cd7af0b2a94dac32bde is 50, key is test_row_0/B:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:05,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742410_1586 (size=12151) 2024-11-20T12:17:05,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:05,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:05,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105086047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/670f82c4dc014cd7af0b2a94dac32bde 2024-11-20T12:17:06,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/f921a2460de24ab78e5ab8746d0bd2be is 50, key is test_row_0/C:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:06,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742411_1587 (size=12151) 2024-11-20T12:17:06,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:06,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:06,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/f921a2460de24ab78e5ab8746d0bd2be 2024-11-20T12:17:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/bed24805c37749c995f612b44d542bf2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2 2024-11-20T12:17:06,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2, entries=200, sequenceid=181, filesize=38.8 K 2024-11-20T12:17:06,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/670f82c4dc014cd7af0b2a94dac32bde as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde 2024-11-20T12:17:06,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde, entries=150, sequenceid=181, filesize=11.9 K 2024-11-20T12:17:06,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/f921a2460de24ab78e5ab8746d0bd2be as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be 2024-11-20T12:17:06,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be, entries=150, sequenceid=181, filesize=11.9 K 2024-11-20T12:17:06,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ddc2f1efcc9e364aaba38798681135ba in 1673ms, sequenceid=181, compaction requested=true 2024-11-20T12:17:06,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:06,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:17:06,492 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:06,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:06,493 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:17:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:17:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:17:06,494 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,494 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,494 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/d940e9a1877b4a188437f9dd908b225a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=47.5 K 2024-11-20T12:17:06,494 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=130.0 K 2024-11-20T12:17:06,494 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2] 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting d940e9a1877b4a188437f9dd908b225a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67a81a6853604c9aa6e578d20aa7522d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:06,494 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b80da6335ad42de9c567759929f72f1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732105021260 2024-11-20T12:17:06,495 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bca0abfea9340319ffff2e59f402fd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732105022522 2024-11-20T12:17:06,495 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 670f82c4dc014cd7af0b2a94dac32bde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023662 2024-11-20T12:17:06,495 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ba1fc773b2444ab0befdebb378820d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732105021260 2024-11-20T12:17:06,495 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82aef01f653b422098aa61364a956fbb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732105022522 2024-11-20T12:17:06,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting bed24805c37749c995f612b44d542bf2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023634 2024-11-20T12:17:06,514 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#495 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:06,514 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/62c83076adb149fa8dd322f22e953f4d is 50, key is test_row_0/B:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:06,515 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:06,521 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201832ac8af487470bb2254b7a4e6c8663_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:06,523 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201832ac8af487470bb2254b7a4e6c8663_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:06,524 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201832ac8af487470bb2254b7a4e6c8663_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:06,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742412_1588 (size=12527) 2024-11-20T12:17:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742413_1589 (size=4469) 2024-11-20T12:17:06,543 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#496 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:06,543 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/d1b509c2ad774447a7d29b75230aeecd is 175, key is test_row_0/A:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742414_1590 (size=31481) 2024-11-20T12:17:06,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,580 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:06,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:06,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120504bd78f18964ce4946bdf4931c2c0eb_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105024852/Put/seqid=0 2024-11-20T12:17:06,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742415_1591 (size=12304) 2024-11-20T12:17:06,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:06,605 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120504bd78f18964ce4946bdf4931c2c0eb_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120504bd78f18964ce4946bdf4931c2c0eb_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:06,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/9283664c2a7d45dbb4b2eb93796d80cf, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:06,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/9283664c2a7d45dbb4b2eb93796d80cf is 175, key is test_row_0/A:col10/1732105024852/Put/seqid=0 2024-11-20T12:17:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742416_1592 (size=31105) 2024-11-20T12:17:06,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:06,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:06,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:06,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105086730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105086731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105086732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105086847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105086847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105086847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:06,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:06,946 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/62c83076adb149fa8dd322f22e953f4d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/62c83076adb149fa8dd322f22e953f4d 2024-11-20T12:17:06,950 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into 62c83076adb149fa8dd322f22e953f4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:06,950 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:06,950 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=12, startTime=1732105026492; duration=0sec 2024-11-20T12:17:06,950 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:06,950 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:06,951 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:06,951 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:06,951 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:06,951 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:06,951 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e120c6053e649a9a023b6ad2614ab1c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=47.5 K 2024-11-20T12:17:06,952 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e120c6053e649a9a023b6ad2614ab1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732105020619 2024-11-20T12:17:06,952 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 220d1e5eefc548c29c19fdc9722d675e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732105021260 2024-11-20T12:17:06,952 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c9e4e5688684db5b6899a1d8dd9ee2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732105022522 2024-11-20T12:17:06,953 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f921a2460de24ab78e5ab8746d0bd2be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023662 2024-11-20T12:17:06,961 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/d1b509c2ad774447a7d29b75230aeecd as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd 2024-11-20T12:17:06,966 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into d1b509c2ad774447a7d29b75230aeecd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:06,966 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:06,966 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=12, startTime=1732105026492; duration=0sec 2024-11-20T12:17:06,966 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:06,966 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:06,968 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:06,968 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/02867450e7644bf3be4e0371accf7613 is 50, key is test_row_0/C:col10/1732105023662/Put/seqid=0 2024-11-20T12:17:06,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742417_1593 (size=12527) 2024-11-20T12:17:07,014 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/9283664c2a7d45dbb4b2eb93796d80cf 2024-11-20T12:17:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/9dea18ca372c44238453c3eb49fe1dc1 is 50, key is test_row_0/B:col10/1732105024852/Put/seqid=0 2024-11-20T12:17:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742418_1594 (size=12151) 2024-11-20T12:17:07,025 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/9dea18ca372c44238453c3eb49fe1dc1 2024-11-20T12:17:07,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/7cc104ac9843406484fef1e1f0ddd776 is 50, key is test_row_0/C:col10/1732105024852/Put/seqid=0 2024-11-20T12:17:07,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742419_1595 (size=12151) 2024-11-20T12:17:07,044 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/7cc104ac9843406484fef1e1f0ddd776 2024-11-20T12:17:07,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/9283664c2a7d45dbb4b2eb93796d80cf as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf 2024-11-20T12:17:07,055 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf, entries=150, sequenceid=200, filesize=30.4 K 2024-11-20T12:17:07,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/9dea18ca372c44238453c3eb49fe1dc1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1 2024-11-20T12:17:07,059 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1, entries=150, sequenceid=200, filesize=11.9 K 2024-11-20T12:17:07,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/7cc104ac9843406484fef1e1f0ddd776 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776 2024-11-20T12:17:07,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105087057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105087057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105087057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,068 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776, entries=150, sequenceid=200, filesize=11.9 K 2024-11-20T12:17:07,069 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ddc2f1efcc9e364aaba38798681135ba in 489ms, sequenceid=200, compaction requested=false 2024-11-20T12:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-20T12:17:07,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-20T12:17:07,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:07,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:07,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-20T12:17:07,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-20T12:17:07,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.5660 sec 2024-11-20T12:17:07,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bb3fb53716504224a6e5e4bf12a6f028_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742420_1596 (size=14794) 2024-11-20T12:17:07,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:07,106 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bb3fb53716504224a6e5e4bf12a6f028_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bb3fb53716504224a6e5e4bf12a6f028_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:07,107 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4ad32611e7f34a859bb7da222b59e01d, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:07,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4ad32611e7f34a859bb7da222b59e01d is 175, key is test_row_0/A:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742421_1597 (size=39749) 2024-11-20T12:17:07,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105087165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105087276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105087366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105087366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105087376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,383 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/02867450e7644bf3be4e0371accf7613 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/02867450e7644bf3be4e0371accf7613 2024-11-20T12:17:07,393 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into 02867450e7644bf3be4e0371accf7613(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:07,393 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:07,393 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=12, startTime=1732105026493; duration=0sec 2024-11-20T12:17:07,393 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:07,393 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:07,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105087487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4ad32611e7f34a859bb7da222b59e01d 2024-11-20T12:17:07,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f1f2ce2b47074abe816347b7b93d8981 is 50, key is test_row_0/B:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742422_1598 (size=12151) 2024-11-20T12:17:07,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f1f2ce2b47074abe816347b7b93d8981 2024-11-20T12:17:07,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/bbf303b8c2524e1a87242165926b4ed9 is 50, key is test_row_0/C:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742423_1599 (size=12151) 2024-11-20T12:17:07,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/bbf303b8c2524e1a87242165926b4ed9 2024-11-20T12:17:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4ad32611e7f34a859bb7da222b59e01d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d 2024-11-20T12:17:07,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d, entries=200, sequenceid=216, filesize=38.8 K 2024-11-20T12:17:07,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f1f2ce2b47074abe816347b7b93d8981 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981 2024-11-20T12:17:07,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T12:17:07,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/bbf303b8c2524e1a87242165926b4ed9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9 2024-11-20T12:17:07,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9, entries=150, sequenceid=216, filesize=11.9 K 2024-11-20T12:17:07,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ddc2f1efcc9e364aaba38798681135ba in 532ms, sequenceid=216, compaction requested=true 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:07,605 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:07,605 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:17:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:07,606 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:07,606 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:17:07,606 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,606 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/62c83076adb149fa8dd322f22e953f4d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=36.0 K 2024-11-20T12:17:07,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:07,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:17:07,606 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,606 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=99.9 K 2024-11-20T12:17:07,606 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,606 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d] 2024-11-20T12:17:07,608 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b509c2ad774447a7d29b75230aeecd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023662 2024-11-20T12:17:07,608 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c83076adb149fa8dd322f22e953f4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023662 2024-11-20T12:17:07,608 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9283664c2a7d45dbb4b2eb93796d80cf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732105024852 2024-11-20T12:17:07,610 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dea18ca372c44238453c3eb49fe1dc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732105024852 2024-11-20T12:17:07,610 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ad32611e7f34a859bb7da222b59e01d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026728 2024-11-20T12:17:07,610 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f2ce2b47074abe816347b7b93d8981, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026729 2024-11-20T12:17:07,630 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:07,631 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/75e3bfd436dd4cd9b566e1877d4a4a24 is 50, key is test_row_0/B:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,637 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:07,645 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112029c8f6b7b2864ad5845daac9bdec4810_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:07,647 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112029c8f6b7b2864ad5845daac9bdec4810_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:07,647 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112029c8f6b7b2864ad5845daac9bdec4810_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:07,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742424_1600 (size=12629) 2024-11-20T12:17:07,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-20T12:17:07,657 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:07,658 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-20T12:17:07,662 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:07,664 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:07,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:07,667 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/75e3bfd436dd4cd9b566e1877d4a4a24 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/75e3bfd436dd4cd9b566e1877d4a4a24 2024-11-20T12:17:07,674 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into 75e3bfd436dd4cd9b566e1877d4a4a24(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:07,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:07,674 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=13, startTime=1732105027605; duration=0sec 2024-11-20T12:17:07,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:07,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:07,674 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:07,675 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:07,675 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:07,675 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,675 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/02867450e7644bf3be4e0371accf7613, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=36.0 K 2024-11-20T12:17:07,675 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 02867450e7644bf3be4e0371accf7613, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732105023662 2024-11-20T12:17:07,676 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cc104ac9843406484fef1e1f0ddd776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732105024852 2024-11-20T12:17:07,676 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting bbf303b8c2524e1a87242165926b4ed9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026729 2024-11-20T12:17:07,686 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:07,687 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/a7fc2a5386bb41bcac359ddc765ea56f is 50, key is test_row_0/C:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:07,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742425_1601 (size=4469) 2024-11-20T12:17:07,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742426_1602 (size=12629) 2024-11-20T12:17:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:07,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:07,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:07,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:07,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6322fb7a98444cb8e9b7e67a0697ec5_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105027798/Put/seqid=0 2024-11-20T12:17:07,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742427_1603 (size=14794) 2024-11-20T12:17:07,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T12:17:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105087854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105087877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105087877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,880 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105087887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105087966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:07,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T12:17:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:07,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:07,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:07,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:08,090 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#505 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:08,091 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/f6a2a34799674732b4c8703cf08a2681 is 175, key is test_row_0/A:col10/1732105026729/Put/seqid=0 2024-11-20T12:17:08,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742428_1604 (size=31583) 2024-11-20T12:17:08,105 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/a7fc2a5386bb41bcac359ddc765ea56f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a7fc2a5386bb41bcac359ddc765ea56f 2024-11-20T12:17:08,108 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into a7fc2a5386bb41bcac359ddc765ea56f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:08,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:08,108 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=13, startTime=1732105027605; duration=0sec 2024-11-20T12:17:08,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:08,108 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:08,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T12:17:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:08,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:08,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105088176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,214 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:08,218 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a6322fb7a98444cb8e9b7e67a0697ec5_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6322fb7a98444cb8e9b7e67a0697ec5_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:08,219 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/081c01d31b6e41a78045fd64b1d8bd02, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:08,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/081c01d31b6e41a78045fd64b1d8bd02 is 175, key is test_row_0/A:col10/1732105027798/Put/seqid=0 2024-11-20T12:17:08,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742429_1605 (size=39749) 2024-11-20T12:17:08,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/081c01d31b6e41a78045fd64b1d8bd02 2024-11-20T12:17:08,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/4d8ba375999f4cc7967f630468cd8d46 is 50, key is test_row_0/B:col10/1732105027798/Put/seqid=0 2024-11-20T12:17:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742430_1606 (size=12151) 2024-11-20T12:17:08,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/4d8ba375999f4cc7967f630468cd8d46 2024-11-20T12:17:08,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/73fab3f35c1140559586d1a348f4afa9 is 50, key is test_row_0/C:col10/1732105027798/Put/seqid=0 2024-11-20T12:17:08,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742431_1607 (size=12151) 2024-11-20T12:17:08,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/73fab3f35c1140559586d1a348f4afa9 2024-11-20T12:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/081c01d31b6e41a78045fd64b1d8bd02 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02 2024-11-20T12:17:08,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02, entries=200, sequenceid=237, filesize=38.8 K 2024-11-20T12:17:08,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/4d8ba375999f4cc7967f630468cd8d46 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46 2024-11-20T12:17:08,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T12:17:08,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/73fab3f35c1140559586d1a348f4afa9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9 2024-11-20T12:17:08,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9, entries=150, sequenceid=237, filesize=11.9 K 2024-11-20T12:17:08,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ddc2f1efcc9e364aaba38798681135ba in 473ms, sequenceid=237, compaction requested=false 2024-11-20T12:17:08,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:08,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:08,276 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:08,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:08,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e72d4f109b0d4347b09c28066030381a_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105027841/Put/seqid=0 2024-11-20T12:17:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:08,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742432_1608 (size=12304) 2024-11-20T12:17:08,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:08,500 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/f6a2a34799674732b4c8703cf08a2681 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681 2024-11-20T12:17:08,505 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into f6a2a34799674732b4c8703cf08a2681(size=30.8 K), total size for store is 69.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:08,505 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:08,505 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=13, startTime=1732105027605; duration=0sec 2024-11-20T12:17:08,506 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:08,506 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105088581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:08,692 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e72d4f109b0d4347b09c28066030381a_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e72d4f109b0d4347b09c28066030381a_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:08,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4b4312500b4444d894c9c31a05d89000, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:08,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4b4312500b4444d894c9c31a05d89000 is 175, key is test_row_0/A:col10/1732105027841/Put/seqid=0 2024-11-20T12:17:08,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105088697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742433_1609 (size=31105) 2024-11-20T12:17:08,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105088886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105088886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,889 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:08,890 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105088897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,899 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105088907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:09,106 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4b4312500b4444d894c9c31a05d89000 2024-11-20T12:17:09,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/242887a1a49147d68bf24f5fad4a8b86 is 50, key is test_row_0/B:col10/1732105027841/Put/seqid=0 2024-11-20T12:17:09,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742434_1610 (size=12151) 2024-11-20T12:17:09,117 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/242887a1a49147d68bf24f5fad4a8b86 2024-11-20T12:17:09,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/df758cdc4781497e9538345c8c37250f is 50, key is test_row_0/C:col10/1732105027841/Put/seqid=0 2024-11-20T12:17:09,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742435_1611 (size=12151) 2024-11-20T12:17:09,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:09,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105089216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:09,526 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/df758cdc4781497e9538345c8c37250f 2024-11-20T12:17:09,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/4b4312500b4444d894c9c31a05d89000 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000 2024-11-20T12:17:09,531 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000, entries=150, sequenceid=251, filesize=30.4 K 2024-11-20T12:17:09,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/242887a1a49147d68bf24f5fad4a8b86 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86 2024-11-20T12:17:09,538 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T12:17:09,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/df758cdc4781497e9538345c8c37250f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f 2024-11-20T12:17:09,542 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-20T12:17:09,542 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ddc2f1efcc9e364aaba38798681135ba in 1266ms, sequenceid=251, compaction requested=true 2024-11-20T12:17:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-20T12:17:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-20T12:17:09,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-20T12:17:09,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-20T12:17:09,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.8870 sec 2024-11-20T12:17:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:09,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:09,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:09,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120284f80520ba24af69eee71ceb12afd6e_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742436_1612 (size=14994) 2024-11-20T12:17:09,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:09,743 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120284f80520ba24af69eee71ceb12afd6e_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120284f80520ba24af69eee71ceb12afd6e_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:09,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/217da3da143c47919f70ee538211a2b4, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:09,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/217da3da143c47919f70ee538211a2b4 is 175, key is test_row_0/A:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:09,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742437_1613 (size=39949) 2024-11-20T12:17:09,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105089778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,783 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T12:17:09,806 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:09,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-20T12:17:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:09,808 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:09,809 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:09,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105089886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:09,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:09,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:09,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:09,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:09,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105090097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:10,148 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/217da3da143c47919f70ee538211a2b4 2024-11-20T12:17:10,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f3baffe704114a72834e0ec633aee397 is 50, key is test_row_0/B:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:10,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742438_1614 (size=12301) 2024-11-20T12:17:10,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105090406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:10,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f3baffe704114a72834e0ec633aee397 2024-11-20T12:17:10,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e114feaba50b4f4aa4b43f8f3f92a126 is 50, key is test_row_0/C:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:10,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742439_1615 (size=12301) 2024-11-20T12:17:10,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:10,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105090897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,897 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105090906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,909 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105090916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105090916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,922 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:10,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e114feaba50b4f4aa4b43f8f3f92a126 2024-11-20T12:17:10,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/217da3da143c47919f70ee538211a2b4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4 2024-11-20T12:17:10,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4, entries=200, sequenceid=272, filesize=39.0 K 2024-11-20T12:17:10,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/f3baffe704114a72834e0ec633aee397 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397 2024-11-20T12:17:10,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T12:17:10,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/e114feaba50b4f4aa4b43f8f3f92a126 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126 2024-11-20T12:17:10,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126, entries=150, sequenceid=272, filesize=12.0 K 2024-11-20T12:17:10,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ddc2f1efcc9e364aaba38798681135ba in 1262ms, sequenceid=272, compaction requested=true 2024-11-20T12:17:10,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:10,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:17:10,992 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:10,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:10,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:17:10,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:10,993 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:10,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:17:10,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:10,993 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:10,993 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:10,993 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:17:10,993 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:17:10,994 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,994 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,994 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/75e3bfd436dd4cd9b566e1877d4a4a24, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=48.1 K 2024-11-20T12:17:10,994 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=139.0 K 2024-11-20T12:17:10,994 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:10,994 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4] 2024-11-20T12:17:10,995 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6a2a34799674732b4c8703cf08a2681, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026729 2024-11-20T12:17:10,995 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 75e3bfd436dd4cd9b566e1877d4a4a24, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026729 2024-11-20T12:17:10,995 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 081c01d31b6e41a78045fd64b1d8bd02, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732105027152 2024-11-20T12:17:10,996 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d8ba375999f4cc7967f630468cd8d46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732105027152 2024-11-20T12:17:10,996 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b4312500b4444d894c9c31a05d89000, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732105027831 2024-11-20T12:17:10,996 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 242887a1a49147d68bf24f5fad4a8b86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732105027831 2024-11-20T12:17:10,996 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217da3da143c47919f70ee538211a2b4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:10,996 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting f3baffe704114a72834e0ec633aee397, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:11,004 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:11,007 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411200a477c6f523342ffa428e47b9ff3a1da_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:11,007 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#517 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:11,007 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5229b1793188455d83f3423611c8137b is 50, key is test_row_0/B:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:11,010 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411200a477c6f523342ffa428e47b9ff3a1da_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:11,010 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200a477c6f523342ffa428e47b9ff3a1da_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:11,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742440_1616 (size=12915) 2024-11-20T12:17:11,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:11,030 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:11,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:11,036 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5229b1793188455d83f3423611c8137b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5229b1793188455d83f3423611c8137b 2024-11-20T12:17:11,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742441_1617 (size=4469) 2024-11-20T12:17:11,040 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#516 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:11,041 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/dc637f7014614edf89ce7c0189b64b69 is 175, key is test_row_0/A:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:11,043 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into 5229b1793188455d83f3423611c8137b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:11,043 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:11,043 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=12, startTime=1732105030993; duration=0sec 2024-11-20T12:17:11,043 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:11,043 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:11,043 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T12:17:11,046 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T12:17:11,046 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:11,046 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:11,046 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a7fc2a5386bb41bcac359ddc765ea56f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=48.1 K 2024-11-20T12:17:11,046 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting a7fc2a5386bb41bcac359ddc765ea56f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732105026729 2024-11-20T12:17:11,047 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fab3f35c1140559586d1a348f4afa9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732105027152 2024-11-20T12:17:11,047 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting df758cdc4781497e9538345c8c37250f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732105027831 2024-11-20T12:17:11,047 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting e114feaba50b4f4aa4b43f8f3f92a126, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207ca21e3dc2544c759447642eca90fcc4_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105029777/Put/seqid=0 2024-11-20T12:17:11,067 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#519 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:11,067 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/cc2589a9e51f4dfeb759e157c7a4819c is 50, key is test_row_0/C:col10/1732105028560/Put/seqid=0 2024-11-20T12:17:11,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742442_1618 (size=31869) 2024-11-20T12:17:11,072 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/dc637f7014614edf89ce7c0189b64b69 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69 2024-11-20T12:17:11,080 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into dc637f7014614edf89ce7c0189b64b69(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:11,080 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:11,080 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=12, startTime=1732105030992; duration=0sec 2024-11-20T12:17:11,080 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:11,080 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:11,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742443_1619 (size=12454) 2024-11-20T12:17:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742444_1620 (size=12915) 2024-11-20T12:17:11,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:11,502 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207ca21e3dc2544c759447642eca90fcc4_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ca21e3dc2544c759447642eca90fcc4_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c19aac72d7654346b82f9079b639508b, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:11,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c19aac72d7654346b82f9079b639508b is 175, key is test_row_0/A:col10/1732105029777/Put/seqid=0 2024-11-20T12:17:11,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742445_1621 (size=31255) 2024-11-20T12:17:11,513 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/cc2589a9e51f4dfeb759e157c7a4819c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cc2589a9e51f4dfeb759e157c7a4819c 2024-11-20T12:17:11,517 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into cc2589a9e51f4dfeb759e157c7a4819c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:11,517 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:11,517 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=12, startTime=1732105030993; duration=0sec 2024-11-20T12:17:11,517 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:11,517 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:11,906 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c19aac72d7654346b82f9079b639508b 2024-11-20T12:17:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5620ae5385ab452a8621a1baa1d5512a is 50, key is test_row_0/B:col10/1732105029777/Put/seqid=0 2024-11-20T12:17:11,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742446_1622 (size=12301) 2024-11-20T12:17:11,914 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5620ae5385ab452a8621a1baa1d5512a 2024-11-20T12:17:11,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/b85ff25dfe3349bfac7742e077f6191e is 50, key is test_row_0/C:col10/1732105029777/Put/seqid=0 2024-11-20T12:17:11,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742447_1623 (size=12301) 2024-11-20T12:17:11,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:11,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:11,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:11,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105091995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:11,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:12,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105092106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:12,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105092317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:12,322 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/b85ff25dfe3349bfac7742e077f6191e 2024-11-20T12:17:12,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c19aac72d7654346b82f9079b639508b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b 2024-11-20T12:17:12,329 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b, entries=150, sequenceid=284, filesize=30.5 K 2024-11-20T12:17:12,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5620ae5385ab452a8621a1baa1d5512a as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a 2024-11-20T12:17:12,338 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T12:17:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/b85ff25dfe3349bfac7742e077f6191e as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e 2024-11-20T12:17:12,342 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e, entries=150, sequenceid=284, filesize=12.0 K 2024-11-20T12:17:12,343 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ddc2f1efcc9e364aaba38798681135ba in 1313ms, sequenceid=284, compaction requested=false 2024-11-20T12:17:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:12,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-20T12:17:12,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-20T12:17:12,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-20T12:17:12,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5350 sec 2024-11-20T12:17:12,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.5400 sec 2024-11-20T12:17:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:12,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T12:17:12,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201ce6329123384e2d8cc0f1b01c623b3d_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:12,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742448_1624 (size=14994) 2024-11-20T12:17:12,643 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:12,646 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201ce6329123384e2d8cc0f1b01c623b3d_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201ce6329123384e2d8cc0f1b01c623b3d_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:12,647 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/22875bd3f6a543ec9debe81fe1fa2bd2, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:12,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/22875bd3f6a543ec9debe81fe1fa2bd2 is 175, key is test_row_0/A:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:12,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742449_1625 (size=39949) 2024-11-20T12:17:12,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=308, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/22875bd3f6a543ec9debe81fe1fa2bd2 2024-11-20T12:17:12,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/0699080697d64e4e9989b0d0056a4087 is 50, key is test_row_0/B:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:12,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742450_1626 (size=12301) 2024-11-20T12:17:12,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105092674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:12,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105092786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:13,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:13,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105092997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:13,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/0699080697d64e4e9989b0d0056a4087 2024-11-20T12:17:13,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/5e15da5fdece4ecc87d8088de0d00d47 is 50, key is test_row_0/C:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:13,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742451_1627 (size=12301) 2024-11-20T12:17:13,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105093306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:13,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/5e15da5fdece4ecc87d8088de0d00d47 2024-11-20T12:17:13,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/22875bd3f6a543ec9debe81fe1fa2bd2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2 2024-11-20T12:17:13,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2, entries=200, sequenceid=308, filesize=39.0 K 2024-11-20T12:17:13,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/0699080697d64e4e9989b0d0056a4087 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087 2024-11-20T12:17:13,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T12:17:13,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/5e15da5fdece4ecc87d8088de0d00d47 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47 2024-11-20T12:17:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47, entries=150, sequenceid=308, filesize=12.0 K 2024-11-20T12:17:13,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ddc2f1efcc9e364aaba38798681135ba in 866ms, sequenceid=308, compaction requested=true 2024-11-20T12:17:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T12:17:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T12:17:13,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ddc2f1efcc9e364aaba38798681135ba:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T12:17:13,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:13,496 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:13,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:13,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/A is initiating minor compaction (all files) 2024-11-20T12:17:13,496 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/A in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:13,496 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=100.7 K 2024-11-20T12:17:13,496 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:13,496 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. files: [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2] 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/B is initiating minor compaction (all files) 2024-11-20T12:17:13,497 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/B in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc637f7014614edf89ce7c0189b64b69, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:13,497 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5229b1793188455d83f3423611c8137b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=36.6 K 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19aac72d7654346b82f9079b639508b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732105029752 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5229b1793188455d83f3423611c8137b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22875bd3f6a543ec9debe81fe1fa2bd2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732105031974 2024-11-20T12:17:13,497 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5620ae5385ab452a8621a1baa1d5512a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732105029752 2024-11-20T12:17:13,498 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 0699080697d64e4e9989b0d0056a4087, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732105031974 2024-11-20T12:17:13,501 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:13,503 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120cdd9d59ad2f44c86aeb50169053855ae_ddc2f1efcc9e364aaba38798681135ba store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:13,503 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:13,503 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6767c6086ed04ba0a53ba57ac0316f66 is 50, key is test_row_0/B:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:13,505 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120cdd9d59ad2f44c86aeb50169053855ae_ddc2f1efcc9e364aaba38798681135ba, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:13,505 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cdd9d59ad2f44c86aeb50169053855ae_ddc2f1efcc9e364aaba38798681135ba because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:13,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742453_1629 (size=4469) 2024-11-20T12:17:13,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742452_1628 (size=13017) 2024-11-20T12:17:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:13,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:13,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:13,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120645aff4406af4761a70efb074175c4aa_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105033818/Put/seqid=0 2024-11-20T12:17:13,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742454_1630 (size=14994) 2024-11-20T12:17:13,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:13,841 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120645aff4406af4761a70efb074175c4aa_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120645aff4406af4761a70efb074175c4aa_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:13,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/e25caa6e762e494882d25f6eb45b3c47, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:13,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/e25caa6e762e494882d25f6eb45b3c47 is 175, key is test_row_0/A:col10/1732105033818/Put/seqid=0 2024-11-20T12:17:13,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742455_1631 (size=39949) 2024-11-20T12:17:13,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105093902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:13,910 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#A#compaction#526 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:13,911 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/158b5c4c2fc04f64ad245de9642a5135 is 175, key is test_row_0/A:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:13,914 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6767c6086ed04ba0a53ba57ac0316f66 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6767c6086ed04ba0a53ba57ac0316f66 2024-11-20T12:17:13,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742456_1632 (size=31971) 2024-11-20T12:17:13,917 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/B of ddc2f1efcc9e364aaba38798681135ba into 6767c6086ed04ba0a53ba57ac0316f66(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:13,917 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:13,917 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/B, priority=13, startTime=1732105033495; duration=0sec 2024-11-20T12:17:13,917 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T12:17:13,917 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:B 2024-11-20T12:17:13,917 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T12:17:13,918 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T12:17:13,918 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1541): ddc2f1efcc9e364aaba38798681135ba/C is initiating minor compaction (all files) 2024-11-20T12:17:13,918 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ddc2f1efcc9e364aaba38798681135ba/C in TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:13,918 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cc2589a9e51f4dfeb759e157c7a4819c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47] into tmpdir=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp, totalSize=36.6 K 2024-11-20T12:17:13,918 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2589a9e51f4dfeb759e157c7a4819c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732105028560 2024-11-20T12:17:13,919 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting b85ff25dfe3349bfac7742e077f6191e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732105029752 2024-11-20T12:17:13,919 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e15da5fdece4ecc87d8088de0d00d47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732105031974 2024-11-20T12:17:13,923 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ddc2f1efcc9e364aaba38798681135ba#C#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T12:17:13,923 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/a9b7e152885840f4a11caf2fa6b786a4 is 50, key is test_row_0/C:col10/1732105031985/Put/seqid=0 2024-11-20T12:17:13,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742457_1633 (size=13017) 2024-11-20T12:17:13,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-20T12:17:13,977 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:13,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T12:17:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-20T12:17:13,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:13,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T12:17:13,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T12:17:13,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T12:17:14,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105094017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:14,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33326 deadline: 1732105094227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105094227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,231 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,247 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/e25caa6e762e494882d25f6eb45b3c47 2024-11-20T12:17:14,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c44e6ff1339049f783fd152a408053d2 is 50, key is test_row_0/B:col10/1732105033818/Put/seqid=0 2024-11-20T12:17:14,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742458_1634 (size=12301) 2024-11-20T12:17:14,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:14,318 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/158b5c4c2fc04f64ad245de9642a5135 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/158b5c4c2fc04f64ad245de9642a5135 2024-11-20T12:17:14,321 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/A of ddc2f1efcc9e364aaba38798681135ba into 158b5c4c2fc04f64ad245de9642a5135(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:14,321 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:14,321 INFO [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/A, priority=13, startTime=1732105033495; duration=0sec 2024-11-20T12:17:14,321 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:14,321 DEBUG [RS:0;843d83d51ba0:42825-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:A 2024-11-20T12:17:14,330 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/a9b7e152885840f4a11caf2fa6b786a4 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a9b7e152885840f4a11caf2fa6b786a4 2024-11-20T12:17:14,333 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ddc2f1efcc9e364aaba38798681135ba/C of ddc2f1efcc9e364aaba38798681135ba into a9b7e152885840f4a11caf2fa6b786a4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T12:17:14,333 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:14,333 INFO [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., storeName=ddc2f1efcc9e364aaba38798681135ba/C, priority=13, startTime=1732105033496; duration=0sec 2024-11-20T12:17:14,333 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T12:17:14,333 DEBUG [RS:0;843d83d51ba0:42825-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ddc2f1efcc9e364aaba38798681135ba:C 2024-11-20T12:17:14,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105094536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:14,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c44e6ff1339049f783fd152a408053d2 2024-11-20T12:17:14,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/17b80af8bbc1448cb4a53fb8dba93ed8 is 50, key is test_row_0/C:col10/1732105033818/Put/seqid=0 2024-11-20T12:17:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742459_1635 (size=12301) 2024-11-20T12:17:14,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:14,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:14,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33306 deadline: 1732105094937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,940 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33350 deadline: 1732105094947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,950 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:14,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33334 deadline: 1732105094956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:14,959 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 843d83d51ba0,42825,1732104864746 for 'test_row_2' in ddc2f1efcc9e364aaba38798681135ba of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T12:17:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:15,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:15,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:15,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:17:15,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:17:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42825 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33342 deadline: 1732105095046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 , the old value is region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T12:17:15,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., hostname=843d83d51ba0,42825,1732104864746, seqNum=5 because the exception is null or not the one we care about 2024-11-20T12:17:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:15,130 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/17b80af8bbc1448cb4a53fb8dba93ed8 2024-11-20T12:17:15,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/e25caa6e762e494882d25f6eb45b3c47 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/e25caa6e762e494882d25f6eb45b3c47 2024-11-20T12:17:15,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/e25caa6e762e494882d25f6eb45b3c47, entries=200, sequenceid=320, filesize=39.0 K 2024-11-20T12:17:15,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/c44e6ff1339049f783fd152a408053d2 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c44e6ff1339049f783fd152a408053d2 2024-11-20T12:17:15,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c44e6ff1339049f783fd152a408053d2, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T12:17:15,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/17b80af8bbc1448cb4a53fb8dba93ed8 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/17b80af8bbc1448cb4a53fb8dba93ed8 2024-11-20T12:17:15,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/17b80af8bbc1448cb4a53fb8dba93ed8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-20T12:17:15,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ddc2f1efcc9e364aaba38798681135ba in 1323ms, sequenceid=320, compaction requested=false 2024-11-20T12:17:15,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:15,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42825 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-20T12:17:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:15,198 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T12:17:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:15,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:15,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203829938cb3764fae80e9000b006a621f_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105033895/Put/seqid=0 2024-11-20T12:17:15,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742460_1636 (size=12454) 2024-11-20T12:17:15,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203829938cb3764fae80e9000b006a621f_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203829938cb3764fae80e9000b006a621f_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:15,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/fb66956e486044158ccd5ed6784cb819, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:15,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/fb66956e486044158ccd5ed6784cb819 is 175, key is test_row_0/A:col10/1732105033895/Put/seqid=0 2024-11-20T12:17:15,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742461_1637 (size=31255) 2024-11-20T12:17:15,907 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-11-20T12:17:15,907 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:15,907 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,907 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:15,908 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-11-20T12:17:15,908 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:15,908 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,908 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:15,909 INFO [Thread-2425 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2425. 2024-11-20T12:17:15,909 DEBUG [Thread-2425 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:15,909 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,909 DEBUG [Thread-2425 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:15,910 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-11-20T12:17:15,910 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:15,910 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,910 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:15,910 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-11-20T12:17:15,911 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:15,911 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,911 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:15,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:16,017 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/fb66956e486044158ccd5ed6784cb819 2024-11-20T12:17:16,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5f221c02bb0f46a09ee8cf8f11041e8f is 50, key is test_row_0/B:col10/1732105033895/Put/seqid=0 2024-11-20T12:17:16,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742462_1638 (size=12301) 2024-11-20T12:17:16,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] regionserver.HRegion(8855): Flush requested on ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:16,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. as already flushing 2024-11-20T12:17:16,058 INFO [Thread-2422 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2422. 2024-11-20T12:17:16,058 DEBUG [Thread-2422 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:16,058 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:16,058 DEBUG [Thread-2422 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:16,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:16,425 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5f221c02bb0f46a09ee8cf8f11041e8f 2024-11-20T12:17:16,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/88fe4f58b33c4132a4c99e4970573c08 is 50, key is test_row_0/C:col10/1732105033895/Put/seqid=0 2024-11-20T12:17:16,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742463_1639 (size=12301) 2024-11-20T12:17:16,837 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/88fe4f58b33c4132a4c99e4970573c08 2024-11-20T12:17:16,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/fb66956e486044158ccd5ed6784cb819 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/fb66956e486044158ccd5ed6784cb819 2024-11-20T12:17:16,847 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/fb66956e486044158ccd5ed6784cb819, entries=150, sequenceid=343, filesize=30.5 K 2024-11-20T12:17:16,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/5f221c02bb0f46a09ee8cf8f11041e8f as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5f221c02bb0f46a09ee8cf8f11041e8f 2024-11-20T12:17:16,851 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5f221c02bb0f46a09ee8cf8f11041e8f, entries=150, sequenceid=343, filesize=12.0 K 2024-11-20T12:17:16,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/88fe4f58b33c4132a4c99e4970573c08 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/88fe4f58b33c4132a4c99e4970573c08 2024-11-20T12:17:16,855 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/88fe4f58b33c4132a4c99e4970573c08, entries=150, sequenceid=343, filesize=12.0 K 2024-11-20T12:17:16,856 INFO [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for ddc2f1efcc9e364aaba38798681135ba in 1658ms, sequenceid=343, compaction requested=true 2024-11-20T12:17:16,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for ddc2f1efcc9e364aaba38798681135ba: 2024-11-20T12:17:16,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:16,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-20T12:17:16,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-20T12:17:16,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-20T12:17:16,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8780 sec 2024-11-20T12:17:16,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 2.8820 sec 2024-11-20T12:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-20T12:17:18,148 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:22,705 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T12:17:24,288 INFO [Thread-2416 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2416. 2024-11-20T12:17:24,288 DEBUG [Thread-2416 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:24,288 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:24,288 DEBUG [Thread-2416 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:24,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:25,008 INFO [Thread-2420 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2420. 2024-11-20T12:17:25,008 DEBUG [Thread-2420 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:25,008 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,008 DEBUG [Thread-2420 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:25,018 INFO [Thread-2418 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2418. 2024-11-20T12:17:25,018 INFO [Thread-2414 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2414. 2024-11-20T12:17:25,018 DEBUG [Thread-2418 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:25,018 DEBUG [Thread-2414 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T12:17:25,018 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,018 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,018 DEBUG [Thread-2418 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,018 DEBUG [Thread-2414 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:25,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2826 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2872 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2851 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2838 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2884 2024-11-20T12:17:25,019 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T12:17:25,019 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:17:25,019 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:17:25,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:25,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:25,020 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T12:17:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:25,023 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105045023"}]},"ts":"1732105045023"} 2024-11-20T12:17:25,025 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T12:17:25,025 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T12:17:25,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T12:17:25,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, UNASSIGN}] 2024-11-20T12:17:25,029 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, UNASSIGN 2024-11-20T12:17:25,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=CLOSING, regionLocation=843d83d51ba0,42825,1732104864746 2024-11-20T12:17:25,031 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40015 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=843d83d51ba0,42825,1732104864746, table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:17:25,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=178, ppid=177, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, UNASSIGN because future has completed 2024-11-20T12:17:25,032 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:17:25,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746}] 2024-11-20T12:17:25,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:25,186 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(122): Close ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:25,186 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:17:25,186 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1722): Closing ddc2f1efcc9e364aaba38798681135ba, disabling compactions & flushes 2024-11-20T12:17:25,186 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:25,186 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:25,186 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. after waiting 0 ms 2024-11-20T12:17:25,186 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:25,187 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(2902): Flushing ddc2f1efcc9e364aaba38798681135ba 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T12:17:25,187 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=A 2024-11-20T12:17:25,187 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:25,187 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=B 2024-11-20T12:17:25,187 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:25,187 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ddc2f1efcc9e364aaba38798681135ba, store=C 2024-11-20T12:17:25,188 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T12:17:25,195 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fa9188a9cf2744c0ac2e9b451750668c_ddc2f1efcc9e364aaba38798681135ba is 50, key is test_row_0/A:col10/1732105036057/Put/seqid=0 2024-11-20T12:17:25,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742464_1640 (size=9914) 2024-11-20T12:17:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:25,602 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:17:25,606 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fa9188a9cf2744c0ac2e9b451750668c_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fa9188a9cf2744c0ac2e9b451750668c_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:25,607 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c83ae373105846d9b6c04529916a0096, store: [table=TestAcidGuarantees family=A region=ddc2f1efcc9e364aaba38798681135ba] 2024-11-20T12:17:25,607 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c83ae373105846d9b6c04529916a0096 is 175, key is test_row_0/A:col10/1732105036057/Put/seqid=0 2024-11-20T12:17:25,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742465_1641 (size=22561) 2024-11-20T12:17:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:26,012 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c83ae373105846d9b6c04529916a0096 2024-11-20T12:17:26,017 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6b783cc14ff84bb49448d85c343b7d3c is 50, key is test_row_0/B:col10/1732105036057/Put/seqid=0 2024-11-20T12:17:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742466_1642 (size=9857) 2024-11-20T12:17:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:26,421 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6b783cc14ff84bb49448d85c343b7d3c 2024-11-20T12:17:26,426 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8ceceb9cbc0d43019fd66ae98e3bd455 is 50, key is test_row_0/C:col10/1732105036057/Put/seqid=0 2024-11-20T12:17:26,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742467_1643 (size=9857) 2024-11-20T12:17:26,430 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8ceceb9cbc0d43019fd66ae98e3bd455 2024-11-20T12:17:26,433 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/A/c83ae373105846d9b6c04529916a0096 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c83ae373105846d9b6c04529916a0096 2024-11-20T12:17:26,436 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c83ae373105846d9b6c04529916a0096, entries=100, sequenceid=351, filesize=22.0 K 2024-11-20T12:17:26,437 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/B/6b783cc14ff84bb49448d85c343b7d3c as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6b783cc14ff84bb49448d85c343b7d3c 2024-11-20T12:17:26,440 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6b783cc14ff84bb49448d85c343b7d3c, entries=100, sequenceid=351, filesize=9.6 K 2024-11-20T12:17:26,441 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/.tmp/C/8ceceb9cbc0d43019fd66ae98e3bd455 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8ceceb9cbc0d43019fd66ae98e3bd455 2024-11-20T12:17:26,444 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8ceceb9cbc0d43019fd66ae98e3bd455, entries=100, sequenceid=351, filesize=9.6 K 2024-11-20T12:17:26,444 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ddc2f1efcc9e364aaba38798681135ba in 1258ms, sequenceid=351, compaction requested=true 2024-11-20T12:17:26,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2] to archive 2024-11-20T12:17:26,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:17:26,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c82540f404c3454b9b141a9141c6e314 2024-11-20T12:17:26,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/938e66eef9a84e8bb7110bb10c08e671 2024-11-20T12:17:26,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/8c143199e4f94bb2bea9bdefe9bf17d2 2024-11-20T12:17:26,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/b70df3d31c6949db90ce5fed743039e0 2024-11-20T12:17:26,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/5ee78531322e4937935880132a259c80 2024-11-20T12:17:26,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/ee6e91fa17ce4e49a792f59ad46c38b7 2024-11-20T12:17:26,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/2c42592958f74cef8daace275e62fe61 2024-11-20T12:17:26,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/67a81a6853604c9aa6e578d20aa7522d 2024-11-20T12:17:26,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/99ba1fc773b2444ab0befdebb378820d 2024-11-20T12:17:26,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/82aef01f653b422098aa61364a956fbb 2024-11-20T12:17:26,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/bed24805c37749c995f612b44d542bf2 2024-11-20T12:17:26,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/d1b509c2ad774447a7d29b75230aeecd 2024-11-20T12:17:26,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/9283664c2a7d45dbb4b2eb93796d80cf 2024-11-20T12:17:26,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4ad32611e7f34a859bb7da222b59e01d 2024-11-20T12:17:26,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/f6a2a34799674732b4c8703cf08a2681 2024-11-20T12:17:26,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/081c01d31b6e41a78045fd64b1d8bd02 2024-11-20T12:17:26,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/4b4312500b4444d894c9c31a05d89000 2024-11-20T12:17:26,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/217da3da143c47919f70ee538211a2b4 2024-11-20T12:17:26,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/dc637f7014614edf89ce7c0189b64b69 2024-11-20T12:17:26,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c19aac72d7654346b82f9079b639508b 2024-11-20T12:17:26,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/22875bd3f6a543ec9debe81fe1fa2bd2 2024-11-20T12:17:26,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/39d21ec7c612414d924b4ad74ecf337a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/d940e9a1877b4a188437f9dd908b225a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/62c83076adb149fa8dd322f22e953f4d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/75e3bfd436dd4cd9b566e1877d4a4a24, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5229b1793188455d83f3423611c8137b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087] to archive 2024-11-20T12:17:26,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:17:26,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/3fdd30c484e8450f959a59221097b998 2024-11-20T12:17:26,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c9b38442056a48318bac37f5efeb63a0 2024-11-20T12:17:26,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/39d21ec7c612414d924b4ad74ecf337a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/39d21ec7c612414d924b4ad74ecf337a 2024-11-20T12:17:26,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/86c780967e284ae2a91277658bea8ad1 2024-11-20T12:17:26,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/b3c41193e0b449a2b88a321307c38b6f 2024-11-20T12:17:26,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/57d4f96c53854107b0c2dbbc25a7ba91 2024-11-20T12:17:26,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/d940e9a1877b4a188437f9dd908b225a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/d940e9a1877b4a188437f9dd908b225a 2024-11-20T12:17:26,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c8ff8d906ec1413b9d4385877711255a 2024-11-20T12:17:26,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/1b80da6335ad42de9c567759929f72f1 2024-11-20T12:17:26,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6bca0abfea9340319ffff2e59f402fd6 2024-11-20T12:17:26,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/62c83076adb149fa8dd322f22e953f4d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/62c83076adb149fa8dd322f22e953f4d 2024-11-20T12:17:26,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/670f82c4dc014cd7af0b2a94dac32bde 2024-11-20T12:17:26,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/9dea18ca372c44238453c3eb49fe1dc1 2024-11-20T12:17:26,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/75e3bfd436dd4cd9b566e1877d4a4a24 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/75e3bfd436dd4cd9b566e1877d4a4a24 2024-11-20T12:17:26,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f1f2ce2b47074abe816347b7b93d8981 2024-11-20T12:17:26,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/4d8ba375999f4cc7967f630468cd8d46 2024-11-20T12:17:26,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/242887a1a49147d68bf24f5fad4a8b86 2024-11-20T12:17:26,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5229b1793188455d83f3423611c8137b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5229b1793188455d83f3423611c8137b 2024-11-20T12:17:26,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/f3baffe704114a72834e0ec633aee397 2024-11-20T12:17:26,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5620ae5385ab452a8621a1baa1d5512a 2024-11-20T12:17:26,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/0699080697d64e4e9989b0d0056a4087 2024-11-20T12:17:26,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/28423eb99f9240fe9a25ca23668744f8, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e120c6053e649a9a023b6ad2614ab1c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/02867450e7644bf3be4e0371accf7613, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a7fc2a5386bb41bcac359ddc765ea56f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cc2589a9e51f4dfeb759e157c7a4819c, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47] to archive 2024-11-20T12:17:26,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T12:17:26,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/14d2c9d631be433da95931ab48846490 2024-11-20T12:17:26,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/1f61cd4d7ccd472ca44db637225ba59b 2024-11-20T12:17:26,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/28423eb99f9240fe9a25ca23668744f8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/28423eb99f9240fe9a25ca23668744f8 2024-11-20T12:17:26,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd9d93a52f154e35b28d03ad82e8a26b 2024-11-20T12:17:26,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cd143600671c4fb4b0f22bae4b2c5686 2024-11-20T12:17:26,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e77ca1bc68ec4d1b8128197cebefca63 2024-11-20T12:17:26,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e120c6053e649a9a023b6ad2614ab1c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e120c6053e649a9a023b6ad2614ab1c 2024-11-20T12:17:26,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/fd243e84c879415abd71516f9e97e46a 2024-11-20T12:17:26,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/220d1e5eefc548c29c19fdc9722d675e 2024-11-20T12:17:26,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8c9e4e5688684db5b6899a1d8dd9ee2d 2024-11-20T12:17:26,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/02867450e7644bf3be4e0371accf7613 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/02867450e7644bf3be4e0371accf7613 2024-11-20T12:17:26,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/f921a2460de24ab78e5ab8746d0bd2be 2024-11-20T12:17:26,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/7cc104ac9843406484fef1e1f0ddd776 2024-11-20T12:17:26,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a7fc2a5386bb41bcac359ddc765ea56f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a7fc2a5386bb41bcac359ddc765ea56f 2024-11-20T12:17:26,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/bbf303b8c2524e1a87242165926b4ed9 2024-11-20T12:17:26,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/73fab3f35c1140559586d1a348f4afa9 2024-11-20T12:17:26,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/df758cdc4781497e9538345c8c37250f 2024-11-20T12:17:26,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cc2589a9e51f4dfeb759e157c7a4819c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/cc2589a9e51f4dfeb759e157c7a4819c 2024-11-20T12:17:26,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/e114feaba50b4f4aa4b43f8f3f92a126 2024-11-20T12:17:26,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/b85ff25dfe3349bfac7742e077f6191e 2024-11-20T12:17:26,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/5e15da5fdece4ecc87d8088de0d00d47 2024-11-20T12:17:26,496 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits/354.seqid, newMaxSeqId=354, maxSeqId=4 2024-11-20T12:17:26,496 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. 2024-11-20T12:17:26,496 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] regionserver.HRegion(1676): Region close journal for ddc2f1efcc9e364aaba38798681135ba: Waiting for close lock at 1732105045186Running coprocessor pre-close hooks at 1732105045186Disabling compacts and flushes for region at 1732105045186Disabling writes for close at 1732105045186Obtaining lock to block concurrent updates at 1732105045187 (+1 ms)Preparing flush snapshotting stores in ddc2f1efcc9e364aaba38798681135ba at 1732105045187Finished memstore snapshotting TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732105045188 (+1 ms)Flushing stores of TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba. at 1732105045189 (+1 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/A: creating writer at 1732105045189Flushing mob file ddc2f1efcc9e364aaba38798681135ba/A: appending metadata at 1732105045195 (+6 ms)Flushing mob file ddc2f1efcc9e364aaba38798681135ba/A: closing flushed file at 1732105045195Flushing ddc2f1efcc9e364aaba38798681135ba/A: appending metadata at 1732105045607 (+412 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/A: closing flushed file at 1732105045607Flushing ddc2f1efcc9e364aaba38798681135ba/B: creating writer at 1732105046015 (+408 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/B: appending metadata at 1732105046017 (+2 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/B: closing flushed file at 1732105046017Flushing ddc2f1efcc9e364aaba38798681135ba/C: creating writer at 1732105046424 (+407 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/C: appending metadata at 1732105046426 (+2 ms)Flushing ddc2f1efcc9e364aaba38798681135ba/C: closing flushed file at 1732105046426Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4257e160: reopening flushed file at 1732105046432 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c4cc980: reopening flushed file at 1732105046436 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77717770: reopening flushed file at 1732105046440 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ddc2f1efcc9e364aaba38798681135ba in 1258ms, sequenceid=351, compaction requested=true at 1732105046444 (+4 ms)Writing region close event to WAL at 1732105046494 (+50 ms)Running coprocessor post-close hooks at 1732105046496 (+2 ms)Closed at 1732105046496 2024-11-20T12:17:26,498 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=179}] handler.UnassignRegionHandler(157): Closed ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:26,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=178 updating hbase:meta row=ddc2f1efcc9e364aaba38798681135ba, regionState=CLOSED 2024-11-20T12:17:26,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=179, ppid=178, state=RUNNABLE, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 because future has completed 2024-11-20T12:17:26,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-20T12:17:26,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseRegionProcedure ddc2f1efcc9e364aaba38798681135ba, server=843d83d51ba0,42825,1732104864746 in 1.4680 sec 2024-11-20T12:17:26,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=178, resume processing ppid=177 2024-11-20T12:17:26,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, ppid=177, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ddc2f1efcc9e364aaba38798681135ba, UNASSIGN in 1.4730 sec 2024-11-20T12:17:26,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-20T12:17:26,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4760 sec 2024-11-20T12:17:26,504 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732105046504"}]},"ts":"1732105046504"} 2024-11-20T12:17:26,505 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T12:17:26,506 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T12:17:26,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4860 sec 2024-11-20T12:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-20T12:17:27,178 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:27,179 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T12:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,183 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=180, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-20T12:17:27,185 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=180, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,187 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,191 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C, FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits] 2024-11-20T12:17:27,194 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/158b5c4c2fc04f64ad245de9642a5135 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/158b5c4c2fc04f64ad245de9642a5135 2024-11-20T12:17:27,196 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c83ae373105846d9b6c04529916a0096 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/c83ae373105846d9b6c04529916a0096 2024-11-20T12:17:27,197 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/e25caa6e762e494882d25f6eb45b3c47 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/e25caa6e762e494882d25f6eb45b3c47 2024-11-20T12:17:27,198 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/fb66956e486044158ccd5ed6784cb819 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/A/fb66956e486044158ccd5ed6784cb819 2024-11-20T12:17:27,200 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5f221c02bb0f46a09ee8cf8f11041e8f to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/5f221c02bb0f46a09ee8cf8f11041e8f 2024-11-20T12:17:27,200 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6767c6086ed04ba0a53ba57ac0316f66 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6767c6086ed04ba0a53ba57ac0316f66 2024-11-20T12:17:27,201 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6b783cc14ff84bb49448d85c343b7d3c to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/6b783cc14ff84bb49448d85c343b7d3c 2024-11-20T12:17:27,202 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c44e6ff1339049f783fd152a408053d2 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/B/c44e6ff1339049f783fd152a408053d2 2024-11-20T12:17:27,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/17b80af8bbc1448cb4a53fb8dba93ed8 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/17b80af8bbc1448cb4a53fb8dba93ed8 2024-11-20T12:17:27,204 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/88fe4f58b33c4132a4c99e4970573c08 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/88fe4f58b33c4132a4c99e4970573c08 2024-11-20T12:17:27,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8ceceb9cbc0d43019fd66ae98e3bd455 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/8ceceb9cbc0d43019fd66ae98e3bd455 2024-11-20T12:17:27,205 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a9b7e152885840f4a11caf2fa6b786a4 to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/C/a9b7e152885840f4a11caf2fa6b786a4 2024-11-20T12:17:27,207 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits/354.seqid to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba/recovered.edits/354.seqid 2024-11-20T12:17:27,207 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/default/TestAcidGuarantees/ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,207 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T12:17:27,208 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:17:27,208 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T12:17:27,211 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ad9f5ea8af44e3f8cddb4fc47a417bb_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ad9f5ea8af44e3f8cddb4fc47a417bb_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,212 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201ce6329123384e2d8cc0f1b01c623b3d_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201ce6329123384e2d8cc0f1b01c623b3d_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,213 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120284f80520ba24af69eee71ceb12afd6e_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120284f80520ba24af69eee71ceb12afd6e_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,214 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a262cbef00f4393a53e217e60d412a6_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202a262cbef00f4393a53e217e60d412a6_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,215 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203829938cb3764fae80e9000b006a621f_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411203829938cb3764fae80e9000b006a621f_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,216 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120504bd78f18964ce4946bdf4931c2c0eb_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120504bd78f18964ce4946bdf4931c2c0eb_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,217 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120645aff4406af4761a70efb074175c4aa_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120645aff4406af4761a70efb074175c4aa_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,218 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065bcc5c8ca224720b73eb2cfbae3529f_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112065bcc5c8ca224720b73eb2cfbae3529f_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,219 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ca21e3dc2544c759447642eca90fcc4_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411207ca21e3dc2544c759447642eca90fcc4_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,220 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120830f9dde35724dddac3e4e11914640ea_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120830f9dde35724dddac3e4e11914640ea_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,221 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088f65b52cd4d4fb8a9d83aa964b9c3f4_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112088f65b52cd4d4fb8a9d83aa964b9c3f4_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,222 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e8e0410ac924f1d873395341d074b4a_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209e8e0410ac924f1d873395341d074b4a_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,224 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6322fb7a98444cb8e9b7e67a0697ec5_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a6322fb7a98444cb8e9b7e67a0697ec5_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,225 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ab1a3f65e29549cd8e8d56dce1be70d0_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ab1a3f65e29549cd8e8d56dce1be70d0_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,226 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bb3fb53716504224a6e5e4bf12a6f028_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120bb3fb53716504224a6e5e4bf12a6f028_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,227 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0e1579183dd4329bd33c3407d610a31_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d0e1579183dd4329bd33c3407d610a31_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,229 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e72d4f109b0d4347b09c28066030381a_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e72d4f109b0d4347b09c28066030381a_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,230 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fa9188a9cf2744c0ac2e9b451750668c_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fa9188a9cf2744c0ac2e9b451750668c_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,231 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fadfeff03dac44b2b02d3e2c0a6f2a4d_ddc2f1efcc9e364aaba38798681135ba to hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fadfeff03dac44b2b02d3e2c0a6f2a4d_ddc2f1efcc9e364aaba38798681135ba 2024-11-20T12:17:27,232 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T12:17:27,235 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=180, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,238 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T12:17:27,240 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T12:17:27,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=180, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,242 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T12:17:27,242 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732105047242"}]},"ts":"9223372036854775807"} 2024-11-20T12:17:27,244 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T12:17:27,244 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ddc2f1efcc9e364aaba38798681135ba, NAME => 'TestAcidGuarantees,,1732105012784.ddc2f1efcc9e364aaba38798681135ba.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T12:17:27,244 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T12:17:27,245 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732105047244"}]},"ts":"9223372036854775807"} 2024-11-20T12:17:27,246 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T12:17:27,247 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=180, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T12:17:27,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-20T12:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40015 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-11-20T12:17:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T12:17:27,298 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T12:17:27,311 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=441 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=385 (was 426), ProcessCount=11 (was 11), AvailableMemoryMB=2997 (was 2555) - AvailableMemoryMB LEAK? - 2024-11-20T12:17:27,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T12:17:27,311 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:17:27,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:17:27,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:27,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:27,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:17:27,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T12:17:27,311 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1219616701, stopped=false 2024-11-20T12:17:27,312 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=843d83d51ba0,40015,1732104863721 2024-11-20T12:17:27,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T12:17:27,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T12:17:27,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:17:27,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:17:27,373 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:17:27,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:17:27,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:17:27,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:17:27,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:27,374 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:17:27,374 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '843d83d51ba0,42825,1732104864746' ***** 2024-11-20T12:17:27,374 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T12:17:27,374 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T12:17:27,374 INFO [RS:0;843d83d51ba0:42825 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T12:17:27,374 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(959): stopping server 843d83d51ba0,42825,1732104864746 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;843d83d51ba0:42825. 2024-11-20T12:17:27,375 DEBUG [RS:0;843d83d51ba0:42825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:17:27,375 DEBUG [RS:0;843d83d51ba0:42825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T12:17:27,375 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T12:17:27,375 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T12:17:27,376 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:27,376 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T12:17:27,376 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T12:17:27,376 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T12:17:27,376 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T12:17:27,376 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T12:17:27,376 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T12:17:27,396 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/info/9b8c5bec9b554a4585dd3d45caef92e1 is 95, key is TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9./info:/1732104898417/DeleteFamily/seqid=0 2024-11-20T12:17:27,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742468_1644 (size=6514) 2024-11-20T12:17:27,407 INFO [regionserver/843d83d51ba0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:17:27,576 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:27,776 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:27,800 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/info/9b8c5bec9b554a4585dd3d45caef92e1 2024-11-20T12:17:27,821 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/ns/a682f85cde1a41ee972cde5c74f2b9a7 is 93, key is TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9./ns:/1732104898368/DeleteFamily/seqid=0 2024-11-20T12:17:27,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742469_1645 (size=5906) 2024-11-20T12:17:27,976 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:28,103 INFO [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-20T12:17:28,103 INFO [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T12:17:28,104 INFO [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T12:17:28,177 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:28,225 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/ns/a682f85cde1a41ee972cde5c74f2b9a7 2024-11-20T12:17:28,251 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/rep_barrier/72eda96717b94bbcb61f5420f383fd8b is 102, key is TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9./rep_barrier:/1732104898368/DeleteFamily/seqid=0 2024-11-20T12:17:28,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742470_1646 (size=6025) 2024-11-20T12:17:28,377 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:28,577 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:28,654 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/rep_barrier/72eda96717b94bbcb61f5420f383fd8b 2024-11-20T12:17:28,672 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/table/6df52ea82b50456cb1be2ae655f5c5e9 is 96, key is TestAcidGuarantees,,1732104869910.56b7424cfd47be723aa0ea7e4fa06ec9./table:/1732104898368/DeleteFamily/seqid=0 2024-11-20T12:17:28,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742471_1647 (size=5936) 2024-11-20T12:17:28,777 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:28,978 DEBUG [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T12:17:29,075 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/table/6df52ea82b50456cb1be2ae655f5c5e9 2024-11-20T12:17:29,079 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/info/9b8c5bec9b554a4585dd3d45caef92e1 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/info/9b8c5bec9b554a4585dd3d45caef92e1 2024-11-20T12:17:29,083 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/info/9b8c5bec9b554a4585dd3d45caef92e1, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T12:17:29,083 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/ns/a682f85cde1a41ee972cde5c74f2b9a7 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/ns/a682f85cde1a41ee972cde5c74f2b9a7 2024-11-20T12:17:29,087 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/ns/a682f85cde1a41ee972cde5c74f2b9a7, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T12:17:29,088 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/rep_barrier/72eda96717b94bbcb61f5420f383fd8b as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/rep_barrier/72eda96717b94bbcb61f5420f383fd8b 2024-11-20T12:17:29,092 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/rep_barrier/72eda96717b94bbcb61f5420f383fd8b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T12:17:29,093 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/.tmp/table/6df52ea82b50456cb1be2ae655f5c5e9 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/table/6df52ea82b50456cb1be2ae655f5c5e9 2024-11-20T12:17:29,097 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/table/6df52ea82b50456cb1be2ae655f5c5e9, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T12:17:29,097 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false 2024-11-20T12:17:29,103 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T12:17:29,104 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T12:17:29,104 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T12:17:29,104 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732105047375Running coprocessor pre-close hooks at 1732105047376 (+1 ms)Disabling compacts and flushes for region at 1732105047376Disabling writes for close at 1732105047376Obtaining lock to block concurrent updates at 1732105047376Preparing flush snapshotting stores in 1588230740 at 1732105047376Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732105047376Flushing stores of hbase:meta,,1.1588230740 at 1732105047377 (+1 ms)Flushing 1588230740/info: creating writer at 1732105047377Flushing 1588230740/info: appending metadata at 1732105047396 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732105047396Flushing 1588230740/ns: creating writer at 1732105047804 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732105047820 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732105047820Flushing 1588230740/rep_barrier: creating writer at 1732105048233 (+413 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732105048251 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732105048251Flushing 1588230740/table: creating writer at 1732105048657 (+406 ms)Flushing 1588230740/table: appending metadata at 1732105048672 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732105048672Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@617370c1: reopening flushed file at 1732105049078 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c91d83d: reopening flushed file at 1732105049083 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d8fbc9b: reopening flushed file at 1732105049087 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12370eca: reopening flushed file at 1732105049092 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1721ms, sequenceid=90, compaction requested=false at 1732105049098 (+6 ms)Writing region close event to WAL at 1732105049101 (+3 ms)Running coprocessor post-close hooks at 1732105049104 (+3 ms)Closed at 1732105049104 2024-11-20T12:17:29,104 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T12:17:29,178 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(976): stopping server 843d83d51ba0,42825,1732104864746; all regions closed. 2024-11-20T12:17:29,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741834_1010 (size=25344) 2024-11-20T12:17:29,185 DEBUG [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/oldWALs 2024-11-20T12:17:29,185 INFO [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 843d83d51ba0%2C42825%2C1732104864746.meta:.meta(num 1732104869404) 2024-11-20T12:17:29,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741832_1008 (size=15566945) 2024-11-20T12:17:29,190 DEBUG [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/oldWALs 2024-11-20T12:17:29,190 INFO [RS:0;843d83d51ba0:42825 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 843d83d51ba0%2C42825%2C1732104864746:(num 1732104868260) 2024-11-20T12:17:29,190 DEBUG [RS:0;843d83d51ba0:42825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:17:29,190 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:17:29,190 INFO [RS:0;843d83d51ba0:42825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T12:17:29,190 INFO [RS:0;843d83d51ba0:42825 {}] hbase.ChoreService(370): Chore service for: regionserver/843d83d51ba0:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T12:17:29,190 INFO [RS:0;843d83d51ba0:42825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T12:17:29,191 INFO [regionserver/843d83d51ba0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T12:17:29,191 INFO [RS:0;843d83d51ba0:42825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42825 2024-11-20T12:17:29,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/843d83d51ba0,42825,1732104864746 2024-11-20T12:17:29,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:17:29,215 INFO [RS:0;843d83d51ba0:42825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T12:17:29,225 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [843d83d51ba0,42825,1732104864746] 2024-11-20T12:17:29,235 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/843d83d51ba0,42825,1732104864746 already deleted, retry=false 2024-11-20T12:17:29,236 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 843d83d51ba0,42825,1732104864746 expired; onlineServers=0 2024-11-20T12:17:29,236 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '843d83d51ba0,40015,1732104863721' ***** 2024-11-20T12:17:29,236 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T12:17:29,236 INFO [M:0;843d83d51ba0:40015 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:17:29,236 INFO [M:0;843d83d51ba0:40015 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T12:17:29,236 DEBUG [M:0;843d83d51ba0:40015 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T12:17:29,236 DEBUG [M:0;843d83d51ba0:40015 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T12:17:29,236 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T12:17:29,236 DEBUG [master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104867919 {}] cleaner.HFileCleaner(306): Exit Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104867919,5,FailOnTimeoutGroup] 2024-11-20T12:17:29,236 DEBUG [master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104867906 {}] cleaner.HFileCleaner(306): Exit Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104867906,5,FailOnTimeoutGroup] 2024-11-20T12:17:29,236 INFO [M:0;843d83d51ba0:40015 {}] hbase.ChoreService(370): Chore service for: master/843d83d51ba0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T12:17:29,237 INFO [M:0;843d83d51ba0:40015 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T12:17:29,237 DEBUG [M:0;843d83d51ba0:40015 {}] master.HMaster(1795): Stopping service threads 2024-11-20T12:17:29,237 INFO [M:0;843d83d51ba0:40015 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T12:17:29,237 INFO [M:0;843d83d51ba0:40015 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:17:29,237 ERROR [M:0;843d83d51ba0:40015 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T12:17:29,238 INFO [M:0;843d83d51ba0:40015 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:17:29,238 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T12:17:29,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T12:17:29,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:17:29,246 DEBUG [M:0;843d83d51ba0:40015 {}] zookeeper.ZKUtil(347): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T12:17:29,247 WARN [M:0;843d83d51ba0:40015 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T12:17:29,247 INFO [M:0;843d83d51ba0:40015 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/.lastflushedseqids 2024-11-20T12:17:29,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742472_1648 (size=72) 2024-11-20T12:17:29,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:17:29,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42825-0x101582ab7dc0001, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:17:29,326 INFO [RS:0;843d83d51ba0:42825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T12:17:29,326 INFO [RS:0;843d83d51ba0:42825 {}] regionserver.HRegionServer(1031): Exiting; stopping=843d83d51ba0,42825,1732104864746; zookeeper connection closed. 2024-11-20T12:17:29,326 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@789e4776 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@789e4776 2024-11-20T12:17:29,327 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T12:17:29,661 INFO [M:0;843d83d51ba0:40015 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:17:29,662 INFO [M:0;843d83d51ba0:40015 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T12:17:29,662 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T12:17:29,662 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:17:29,662 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:17:29,662 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T12:17:29,663 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:17:29,663 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=800.96 KB heapSize=982.52 KB 2024-11-20T12:17:29,682 DEBUG [M:0;843d83d51ba0:40015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c15903220e34b798aae9fb48ed7bf02 is 82, key is hbase:meta,,1/info:regioninfo/1732104869484/Put/seqid=0 2024-11-20T12:17:29,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742473_1649 (size=5672) 2024-11-20T12:17:30,086 INFO [M:0;843d83d51ba0:40015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2242 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c15903220e34b798aae9fb48ed7bf02 2024-11-20T12:17:30,117 DEBUG [M:0;843d83d51ba0:40015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0ddf6df4d834945b1ae7d2044819d23 is 2480, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1732104901617/Put/seqid=0 2024-11-20T12:17:30,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742474_1650 (size=47168) 2024-11-20T12:17:30,522 INFO [M:0;843d83d51ba0:40015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800.40 KB at sequenceid=2242 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0ddf6df4d834945b1ae7d2044819d23 2024-11-20T12:17:30,530 INFO [M:0;843d83d51ba0:40015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ddf6df4d834945b1ae7d2044819d23 2024-11-20T12:17:30,547 DEBUG [M:0;843d83d51ba0:40015 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac63d1681b14e1faf4fa64d3e78c68d is 69, key is 843d83d51ba0,42825,1732104864746/rs:state/1732104868007/Put/seqid=0 2024-11-20T12:17:30,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073742475_1651 (size=5156) 2024-11-20T12:17:30,952 INFO [M:0;843d83d51ba0:40015 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2242 (bloomFilter=true), to=hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac63d1681b14e1faf4fa64d3e78c68d 2024-11-20T12:17:30,960 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1c15903220e34b798aae9fb48ed7bf02 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1c15903220e34b798aae9fb48ed7bf02 2024-11-20T12:17:30,965 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1c15903220e34b798aae9fb48ed7bf02, entries=8, sequenceid=2242, filesize=5.5 K 2024-11-20T12:17:30,966 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e0ddf6df4d834945b1ae7d2044819d23 as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0ddf6df4d834945b1ae7d2044819d23 2024-11-20T12:17:30,970 INFO [M:0;843d83d51ba0:40015 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ddf6df4d834945b1ae7d2044819d23 2024-11-20T12:17:30,970 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e0ddf6df4d834945b1ae7d2044819d23, entries=180, sequenceid=2242, filesize=46.1 K 2024-11-20T12:17:30,971 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5ac63d1681b14e1faf4fa64d3e78c68d as hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ac63d1681b14e1faf4fa64d3e78c68d 2024-11-20T12:17:30,974 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36391/user/jenkins/test-data/ac5214c0-ba85-2be0-f93d-4745f4352a91/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5ac63d1681b14e1faf4fa64d3e78c68d, entries=1, sequenceid=2242, filesize=5.0 K 2024-11-20T12:17:30,975 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(3140): Finished flush of dataSize ~800.96 KB/820180, heapSize ~982.23 KB/1005800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2242, compaction requested=false 2024-11-20T12:17:30,976 INFO [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:17:30,977 DEBUG [M:0;843d83d51ba0:40015 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732105049662Disabling compacts and flushes for region at 1732105049662Disabling writes for close at 1732105049662Obtaining lock to block concurrent updates at 1732105049663 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732105049663Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=820180, getHeapSize=1006040, getOffHeapSize=0, getCellsCount=2447 at 1732105049664 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732105049665 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732105049665Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732105049681 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732105049681Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732105050094 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732105050117 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732105050117Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732105050530 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732105050547 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732105050547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79357607: reopening flushed file at 1732105050959 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ab5fc3a: reopening flushed file at 1732105050965 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@703343b9: reopening flushed file at 1732105050970 (+5 ms)Finished flush of dataSize ~800.96 KB/820180, heapSize ~982.23 KB/1005800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2242, compaction requested=false at 1732105050975 (+5 ms)Writing region close event to WAL at 1732105050976 (+1 ms)Closed at 1732105050976 2024-11-20T12:17:30,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39791 is added to blk_1073741830_1006 (size=967149) 2024-11-20T12:17:30,980 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T12:17:30,980 INFO [M:0;843d83d51ba0:40015 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T12:17:30,980 INFO [M:0;843d83d51ba0:40015 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40015 2024-11-20T12:17:30,980 INFO [M:0;843d83d51ba0:40015 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T12:17:31,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:17:31,094 INFO [M:0;843d83d51ba0:40015 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T12:17:31,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40015-0x101582ab7dc0000, quorum=127.0.0.1:61195, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:17:31,098 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3dfb1097{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T12:17:31,100 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@16142ff{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T12:17:31,100 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T12:17:31,100 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5ef5f873{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T12:17:31,100 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b8e8cdd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.log.dir/,STOPPED} 2024-11-20T12:17:31,103 WARN [BP-788036131-172.17.0.3-1732104859763 heartbeating to localhost/127.0.0.1:36391 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T12:17:31,103 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T12:17:31,103 WARN [BP-788036131-172.17.0.3-1732104859763 heartbeating to localhost/127.0.0.1:36391 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-788036131-172.17.0.3-1732104859763 (Datanode Uuid b670e9a4-6ad7-405c-9da8-9f4952936e79) service to localhost/127.0.0.1:36391 2024-11-20T12:17:31,103 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T12:17:31,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data1/current/BP-788036131-172.17.0.3-1732104859763 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T12:17:31,106 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/cluster_cd1278b7-4776-fb4d-e087-292e44d2ff8d/data/data2/current/BP-788036131-172.17.0.3-1732104859763 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T12:17:31,107 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T12:17:31,115 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T12:17:31,116 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T12:17:31,116 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T12:17:31,116 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T12:17:31,116 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/2bc69820-76da-95eb-144c-736cf9aabc75/hadoop.log.dir/,STOPPED} 2024-11-20T12:17:31,130 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T12:17:31,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down